diff options
Diffstat (limited to 'python/src/com/jetbrains/python/sdk')
8 files changed, 188 insertions, 230 deletions
diff --git a/python/src/com/jetbrains/python/sdk/CreateVirtualEnvDialog.java b/python/src/com/jetbrains/python/sdk/CreateVirtualEnvDialog.java index 39fc2c909cf6..1d0a4bc3ad54 100644 --- a/python/src/com/jetbrains/python/sdk/CreateVirtualEnvDialog.java +++ b/python/src/com/jetbrains/python/sdk/CreateVirtualEnvDialog.java @@ -48,9 +48,9 @@ import com.intellij.ui.components.JBLabel; import com.intellij.util.NullableConsumer; import com.intellij.util.PathUtil; import com.intellij.util.PlatformUtils; +import com.intellij.webcore.packaging.PackagesNotificationPanel; import com.jetbrains.python.packaging.PyExternalProcessException; import com.jetbrains.python.packaging.PyPackageManager; -import com.jetbrains.python.packaging.PyPackageManagerImpl; import com.jetbrains.python.packaging.PyPackageService; import com.jetbrains.python.sdk.flavors.VirtualEnvSdkFlavor; import com.jetbrains.python.ui.IdeaDialog; @@ -416,7 +416,7 @@ public class CreateVirtualEnvDialog extends IdeaDialog { String myPath; public void run(@NotNull final ProgressIndicator indicator) { - final PyPackageManagerImpl packageManager = (PyPackageManagerImpl)PyPackageManager.getInstance(basicSdk); + final PyPackageManager packageManager = PyPackageManager.getInstance(basicSdk); try { indicator.setText("Creating virtual environment for " + basicSdk.getName()); myPath = packageManager.createVirtualEnv(getDestination(), useGlobalSitePackages()); @@ -425,7 +425,7 @@ public class CreateVirtualEnvDialog extends IdeaDialog { ApplicationManager.getApplication().invokeLater(new Runnable() { @Override public void run() { - packageManager.showInstallationError(getOwner(), "Failed to Create Virtual Environment", e.toString()); + PackagesNotificationPanel.showError(getOwner(), "Failed to Create Virtual Environment", e.toString()); } }, ModalityState.any()); } diff --git a/python/src/com/jetbrains/python/sdk/PySdkUtil.java b/python/src/com/jetbrains/python/sdk/PySdkUtil.java index 8f2b62b1b414..97d5ee545cbb 100644 --- a/python/src/com/jetbrains/python/sdk/PySdkUtil.java +++ b/python/src/com/jetbrains/python/sdk/PySdkUtil.java @@ -15,10 +15,13 @@ */ package com.jetbrains.python.sdk; +import com.intellij.execution.ExecutionException; import com.intellij.execution.process.CapturingProcessHandler; import com.intellij.execution.process.ProcessOutput; +import com.intellij.execution.util.ExecUtil; import com.intellij.openapi.application.PathManager; import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.module.Module; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.roots.OrderRootType; import com.intellij.openapi.util.SystemInfo; @@ -29,8 +32,10 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.remote.RemoteSdkAdditionalData; -import com.intellij.util.ArrayUtil; +import com.intellij.util.SystemProperties; import com.intellij.util.containers.HashMap; +import com.jetbrains.python.packaging.PyPackageUtil; +import com.jetbrains.python.packaging.PyRequirement; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -38,8 +43,7 @@ import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -56,6 +60,7 @@ public class PySdkUtil { // Windows EOF marker, Ctrl+Z public static final int SUBSTITUTE = 26; + public static final String PATH_ENV_VARIABLE = "PATH"; private PySdkUtil() { // explicitly none @@ -87,57 +92,29 @@ public class PySdkUtil { return getProcessOutput(homePath, command, null, timeout); } - /** - * Executes a process and returns its stdout and stderr outputs as lists of lines. - * Waits for process for possibly limited duration. - * - * @param homePath process run directory - * @param command command to execute and its arguments - * @param addEnv items are prepended to same-named values of inherited process environment. - * @param timeout how many milliseconds to wait until the process terminates; non-positive means inifinity. - * @return a tuple of (stdout lines, stderr lines, exit_code), lines in them have line terminators stripped, or may be null. - */ @NotNull public static ProcessOutput getProcessOutput(String homePath, @NonNls String[] command, - @Nullable @NonNls String[] addEnv, + @Nullable @NonNls Map<String, String> extraEnv, final int timeout) { - return getProcessOutput(homePath, command, addEnv, timeout, null, true); + return getProcessOutput(homePath, command, extraEnv, timeout, null, true); } - /** - * Executes a process and returns its stdout and stderr outputs as lists of lines. - * Waits for process for possibly limited duration. - * - * @param homePath process run directory - * @param command command to execute and its arguments - * @param addEnv items are prepended to same-named values of inherited process environment. - * @param timeout how many milliseconds to wait until the process terminates; non-positive means infinity. - * @param stdin the data to write to the process standard input stream - * @param needEOFMarker - * @return a tuple of (stdout lines, stderr lines, exit_code), lines in them have line terminators stripped, or may be null. - */ @NotNull public static ProcessOutput getProcessOutput(String homePath, @NonNls String[] command, - @Nullable @NonNls String[] addEnv, + @Nullable @NonNls Map<String, String> extraEnv, final int timeout, @Nullable byte[] stdin, boolean needEOFMarker) { - final ProcessOutput failureOutput = new ProcessOutput(); if (homePath == null || !new File(homePath).exists()) { - return failureOutput; + return new ProcessOutput(); } + final Map<String, String> systemEnv = System.getenv(); + final Map<String, String> env = extraEnv != null ? mergeEnvVariables(systemEnv, extraEnv) : systemEnv; try { - List<String> commands = new ArrayList<String>(); - if (SystemInfo.isWindows && StringUtil.endsWithIgnoreCase(command[0], ".bat")) { - commands.add("cmd"); - commands.add("/c"); - } - Collections.addAll(commands, command); - String[] newEnv = buildAdditionalEnv(addEnv); - Process process = Runtime.getRuntime().exec(ArrayUtil.toStringArray(commands), newEnv, new File(homePath)); - CapturingProcessHandler processHandler = new CapturingProcessHandler(process); + final Process process = ExecUtil.exec(Arrays.asList(command), homePath, env); + final CapturingProcessHandler processHandler = new CapturingProcessHandler(process); if (stdin != null) { final OutputStream processInput = processHandler.getProcessInput(); assert processInput != null; @@ -152,72 +129,61 @@ public class PySdkUtil { } return processHandler.runProcess(timeout); } - catch (final IOException ex) { - LOG.warn(ex); - return new ProcessOutput() { - @Override - public String getStderr() { - String err = super.getStderr(); - if (!StringUtil.isEmpty(err)) { - err += "\n" + ex.getMessage(); - } - else { - err = ex.getMessage(); - } - return err; - } - }; + catch (ExecutionException e) { + return getOutputForException(e); + } + catch (IOException e) { + return getOutputForException(e); } } - private static String[] buildAdditionalEnv(String[] addEnv) { - String[] newEnv = null; - if (addEnv != null) { - Map<String, String> envMap = buildEnvMap(addEnv); - newEnv = new String[envMap.size()]; - int i = 0; - for (Map.Entry<String, String> entry : envMap.entrySet()) { - newEnv[i] = entry.getKey() + "=" + entry.getValue(); - i += 1; + private static ProcessOutput getOutputForException(final Exception e) { + LOG.warn(e); + return new ProcessOutput() { + @Override + public String getStderr() { + String err = super.getStderr(); + if (!StringUtil.isEmpty(err)) { + err += "\n" + e.getMessage(); + } + else { + err = e.getMessage(); + } + return err; } - } - return newEnv; + }; } - public static Map<String, String> buildEnvMap(String[] addEnv) { - Map<String, String> envMap = new HashMap<String, String>(System.getenv()); - // turn additional ent into map - Map<String, String> addMap = new HashMap<String, String>(); - for (String envItem : addEnv) { - int pos = envItem.indexOf('='); - if (pos > 0) { - String key = envItem.substring(0, pos); - String value = envItem.substring(pos + 1, envItem.length()); - addMap.put(key, value); - } - else { - LOG.warn(String.format("Invalid env value: '%s'", envItem)); - } - } - // fuse old and new - for (Map.Entry<String, String> entry : addMap.entrySet()) { - final String key = entry.getKey(); - final String value = entry.getValue(); - final String oldValue = envMap.get(key); - if (oldValue != null) { - envMap.put(key, value + oldValue); + @NotNull + public static Map<String, String> mergeEnvVariables(@NotNull Map<String, String> environment, + @NotNull Map<String, String> extraEnvironment) { + final Map<String, String> result = new HashMap<String, String>(environment); + for (Map.Entry<String, String> entry : extraEnvironment.entrySet()) { + if (PATH_ENV_VARIABLE.equals(entry.getKey()) && result.containsKey(PATH_ENV_VARIABLE)) { + result.put(PATH_ENV_VARIABLE, result.get(PATH_ENV_VARIABLE) + File.pathSeparator + entry.getValue()); } else { - envMap.put(key, value); + result.put(entry.getKey(), entry.getValue()); } } - return envMap; + return result; } public static boolean isRemote(@Nullable Sdk sdk) { return sdk != null && sdk.getSdkAdditionalData() instanceof RemoteSdkAdditionalData; } + public static String getUserSite() { + if (SystemInfo.isWindows) { + final String appdata = System.getenv("APPDATA"); + return appdata + File.separator + "Python"; + } + else { + final String userHome = SystemProperties.getUserHome(); + return userHome + File.separator + ".local"; + } + } + public static boolean isElementInSkeletons(@NotNull final PsiElement element) { final PsiFile file = element.getContainingFile(); if (file != null) { @@ -266,4 +232,13 @@ public class PySdkUtil { } return null; } + + @Nullable + public static List<PyRequirement> getRequirementsFromTxt(Module module) { + final VirtualFile requirementsTxt = PyPackageUtil.findRequirementsTxt(module); + if (requirementsTxt != null) { + return PyRequirement.parse(requirementsTxt); + } + return null; + } } diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkDetailsStep.java b/python/src/com/jetbrains/python/sdk/PythonSdkDetailsStep.java index 923629ce4473..cc6a2c6b6121 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkDetailsStep.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkDetailsStep.java @@ -46,7 +46,7 @@ import java.util.List; import java.util.Set; public class PythonSdkDetailsStep extends BaseListPopupStep<String> { - private DialogWrapper myMore; + @Nullable private DialogWrapper myMore; private final Project myProject; private final Component myOwnerComponent; private final Sdk[] myExistingSdks; @@ -120,6 +120,8 @@ public class PythonSdkDetailsStep extends BaseListPopupStep<String> { } private void optionSelected(final String selectedValue) { + if (!MORE.equals(selectedValue) && myMore != null) + Disposer.dispose(myMore.getDisposable()); if (LOCAL.equals(selectedValue)) { createLocalSdk(); } @@ -129,7 +131,7 @@ public class PythonSdkDetailsStep extends BaseListPopupStep<String> { else if (VIRTUALENV.equals(selectedValue)) { createVirtualEnvSdk(); } - else { + else if (myMore != null) { myMore.show(); } } diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkType.java b/python/src/com/jetbrains/python/sdk/PythonSdkType.java index 58de09370d37..460c875491ce 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkType.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkType.java @@ -15,6 +15,7 @@ */ package com.jetbrains.python.sdk; +import com.google.common.collect.ImmutableMap; import com.intellij.execution.ExecutionException; import com.intellij.execution.configurations.GeneralCommandLine; import com.intellij.execution.process.ProcessOutput; @@ -771,16 +772,12 @@ public class PythonSdkType extends SdkType { } @NotNull - public static List<String> getSysPathsFromScript(String bin_path) throws InvalidSdkException { + public static List<String> getSysPathsFromScript(@NotNull String binaryPath) throws InvalidSdkException { String scriptFile = PythonHelpersLocator.getHelperPath("syspath.py"); // to handle the situation when PYTHONPATH contains ., we need to run the syspath script in the // directory of the script itself - otherwise the dir in which we run the script (e.g. /usr/bin) will be added to SDK path - String[] add_environment = getVirtualEnvAdditionalEnv(bin_path); - final ProcessOutput run_result = PySdkUtil.getProcessOutput( - new File(scriptFile).getParent(), - new String[]{bin_path, scriptFile}, - add_environment, MINUTE - ); + final ProcessOutput run_result = PySdkUtil.getProcessOutput(new File(scriptFile).getParent(), new String[]{binaryPath, scriptFile}, + getVirtualEnvExtraEnv(binaryPath), MINUTE); if (!run_result.checkSuccess(LOG)) { throw new InvalidSdkException(String.format("Failed to determine Python's sys.path value:\nSTDOUT: %s\nSTDERR: %s", run_result.getStdout(), @@ -789,15 +786,16 @@ public class PythonSdkType extends SdkType { return run_result.getStdoutLines(); } - // Returns a piece of env good as additional env for getProcessOutput. + /** + * Returns a piece of env good as additional env for getProcessOutput. + */ @Nullable - public static String[] getVirtualEnvAdditionalEnv(String bin_path) { - File virtualenv_root = getVirtualEnvRoot(bin_path); - String[] add_environment = null; - if (virtualenv_root != null) { - add_environment = new String[]{"PATH=" + virtualenv_root + File.pathSeparator}; + public static Map<String, String> getVirtualEnvExtraEnv(@NotNull String binaryPath) { + final File root = getVirtualEnvRoot(binaryPath); + if (root != null) { + return ImmutableMap.of("PATH", root.toString()); } - return add_environment; + return null; } @Nullable diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java index b4b968fdb108..fb77cdb26f8f 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java @@ -32,7 +32,6 @@ import com.intellij.openapi.projectRoots.SdkModificator; import com.intellij.openapi.projectRoots.SdkTypeId; import com.intellij.openapi.roots.OrderRootType; import com.intellij.openapi.startup.StartupActivity; -import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.io.FileUtilRt; import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VirtualFile; @@ -149,7 +148,7 @@ public class PythonSdkUpdater implements StartupActivity { } public static void updateSdk(@Nullable Project project, @Nullable Component ownerComponent, @NotNull final Sdk sdk, String skeletonsPath) throws InvalidSdkException { - PySkeletonRefresher.refreshSkeletonsOfSdk(project, ownerComponent, skeletonsPath, new Ref<Boolean>(false), sdk); // NOTE: whole thing would need a rename + PySkeletonRefresher.refreshSkeletonsOfSdk(project, ownerComponent, skeletonsPath, sdk); // NOTE: whole thing would need a rename if (!PySdkUtil.isRemote(sdk)) { updateSysPath(sdk); } diff --git a/python/src/com/jetbrains/python/sdk/flavors/WinPythonSdkFlavor.java b/python/src/com/jetbrains/python/sdk/flavors/WinPythonSdkFlavor.java index 9d22af8c03bf..3a9ac951509d 100644 --- a/python/src/com/jetbrains/python/sdk/flavors/WinPythonSdkFlavor.java +++ b/python/src/com/jetbrains/python/sdk/flavors/WinPythonSdkFlavor.java @@ -37,6 +37,8 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor { "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Python\\PythonCore", "python.exe", "HKEY_LOCAL_MACHINE\\SOFTWARE\\IronPython", "ipy.exe"); + private static Set<String> ourRegistryCache; + private WinPythonSdkFlavor() { } @@ -78,18 +80,25 @@ public class WinPythonSdkFlavor extends CPythonSdkFlavor { } public static void findInRegistry(Collection<String> candidates) { - for (Map.Entry<String, String> entry : ourRegistryMap.entrySet()) { - final String prefix = entry.getKey(); - final String exePath = entry.getValue(); - List<String> strings = WindowsRegistryUtil.readRegistryBranch(prefix); - for (String string : strings) { - final String path = - WindowsRegistryUtil.readRegistryDefault(prefix + "\\" + string + - "\\InstallPath"); - if (path != null) { - File f = new File(path, exePath); - if (f.exists()) { - candidates.add(FileUtil.toSystemDependentName(f.getPath())); + fillRegistryCache(); + candidates.addAll(ourRegistryCache); + } + + private static void fillRegistryCache() { + if (ourRegistryCache == null) { + ourRegistryCache = new HashSet<String>(); + for (Map.Entry<String, String> entry : ourRegistryMap.entrySet()) { + final String prefix = entry.getKey(); + final String exePath = entry.getValue(); + List<String> strings = WindowsRegistryUtil.readRegistryBranch(prefix); + for (String string : strings) { + final String path = WindowsRegistryUtil.readRegistryDefault(prefix + "\\" + string + + "\\InstallPath"); + if (path != null) { + File f = new File(path, exePath); + if (f.exists()) { + ourRegistryCache.add(FileUtil.toSystemDependentName(f.getPath())); + } } } } diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java index 901472e6d2c3..7e099041e16a 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonGenerator.java @@ -15,6 +15,7 @@ */ package com.jetbrains.python.sdk.skeletons; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.intellij.execution.process.ProcessOutput; import com.intellij.openapi.application.ex.ApplicationManagerEx; @@ -52,21 +53,12 @@ public class PySkeletonGenerator { ENV_PATH_PARAM.put(IronPythonSdkFlavor.class, "IRONPYTHONPATH"); // TODO: Make strategy and move to PythonSdkFlavor? } - protected static final Logger LOG = Logger.getInstance("#" + PySkeletonGenerator.class.getName()); - - protected static final int MINUTE = 60 * 1000; - protected static final String GENERATOR3 = "generator3.py"; - private static final String[] EMPTY_ENVS = new String[0]; private final String mySkeletonsPath; - /** - * Env variables to be added to skeleton generator - */ - @NotNull - private final String[] myEnvs; + @NotNull private final Map<String, String> myEnv; public void finishSkeletonsGeneration() { } @@ -85,7 +77,6 @@ public class PySkeletonGenerator { } } - /** * @param skeletonPath path where skeletons should be generated * @param pySdk SDK @@ -94,11 +85,11 @@ public class PySkeletonGenerator { public PySkeletonGenerator(String skeletonPath, @NotNull final Sdk pySdk, @Nullable final String currentFolder) { mySkeletonsPath = skeletonPath; final PythonSdkFlavor flavor = PythonSdkFlavor.getFlavor(pySdk); - if ((currentFolder != null) && (flavor != null) && ENV_PATH_PARAM.containsKey(flavor.getClass())) { - myEnvs = new String[]{String.format("%s=%s", ENV_PATH_PARAM.get(flavor.getClass()), currentFolder)}; + if (currentFolder != null && flavor != null && ENV_PATH_PARAM.containsKey(flavor.getClass())) { + myEnv = ImmutableMap.of(ENV_PATH_PARAM.get(flavor.getClass()), currentFolder); } else { - myEnvs = EMPTY_ENVS; + myEnv = Collections.emptyMap(); } } @@ -171,23 +162,19 @@ public class PySkeletonGenerator { if (modfilename != null) { commandLine.add(modfilename); } - final List<String> envs = new ArrayList<String>(Arrays.asList(myEnvs)); - final String[] virtualEnvAdditionalEnv = PythonSdkType.getVirtualEnvAdditionalEnv(binaryPath); - if (virtualEnvAdditionalEnv != null) { - envs.addAll(Arrays.asList(virtualEnvAdditionalEnv)); - } + final Map<String, String> extraEnv = PythonSdkType.getVirtualEnvExtraEnv(binaryPath); + final Map<String, String> env = extraEnv != null ? PySdkUtil.mergeEnvVariables(myEnv, extraEnv) : myEnv; - return getProcessOutput(parent_dir, ArrayUtil.toStringArray(commandLine), envs.toArray(new String[envs.size()]), - MINUTE * 10 - ); + return getProcessOutput(parent_dir, ArrayUtil.toStringArray(commandLine), env, MINUTE * 10); } - protected ProcessOutput getProcessOutput(String homePath, String[] commandLine, String[] env, int timeout) throws InvalidSdkException { + protected ProcessOutput getProcessOutput(String homePath, String[] commandLine, Map<String, String> extraEnv, + int timeout) throws InvalidSdkException { return PySdkUtil.getProcessOutput( homePath, commandLine, - env, + extraEnv, timeout ); } @@ -207,7 +194,7 @@ public class PySkeletonGenerator { "-d", mySkeletonsPath, // output dir "-b", // for builtins }, - PythonSdkType.getVirtualEnvAdditionalEnv(binaryPath), MINUTE * 5 + PythonSdkType.getVirtualEnvExtraEnv(binaryPath), MINUTE * 5 ); runResult.checkSuccess(LOG); LOG.info("Rebuilding builtin skeletons took " + (System.currentTimeMillis() - startTime) + " ms"); @@ -228,7 +215,7 @@ public class PySkeletonGenerator { final ProcessOutput process = getProcessOutput(parentDir, ArrayUtil.toStringArray(cmd), - PythonSdkType.getVirtualEnvAdditionalEnv(homePath), + PythonSdkType.getVirtualEnvExtraEnv(homePath), MINUTE * 4); // see PY-3898 LOG.info("Retrieving binary module list took " + (System.currentTimeMillis() - startTime) + " ms"); diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java index 4e4a43102202..b1e8ca19acf8 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java +++ b/python/src/com/jetbrains/python/sdk/skeletons/PySkeletonRefresher.java @@ -19,9 +19,6 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer; import com.intellij.execution.ExecutionException; -import com.intellij.notification.Notification; -import com.intellij.notification.NotificationType; -import com.intellij.notification.Notifications; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.PathManager; import com.intellij.openapi.diagnostic.Logger; @@ -31,7 +28,6 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.roots.OrderRootType; import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; @@ -48,7 +44,6 @@ import com.jetbrains.python.PyNames; import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil; import com.jetbrains.python.packaging.PyExternalProcessException; import com.jetbrains.python.packaging.PyPackageManager; -import com.jetbrains.python.packaging.PyPackageManagerImpl; import com.jetbrains.python.psi.resolve.PythonSdkPathCache; import com.jetbrains.python.remote.PythonRemoteInterpreterManager; import com.jetbrains.python.sdk.InvalidSdkException; @@ -105,10 +100,6 @@ public class PySkeletonRefresher { private PySkeletonGenerator mySkeletonsGenerator; - public static void refreshSkeletonsOfSdk(@NotNull Project project, @NotNull Sdk sdk) throws InvalidSdkException { - refreshSkeletonsOfSdk(project, null, PythonSdkType.findSkeletonsPath(sdk), new Ref<Boolean>(false), sdk); - } - public static synchronized boolean isGeneratingSkeletons() { return ourGeneratingCount > 0; } @@ -120,7 +111,6 @@ public class PySkeletonRefresher { public static void refreshSkeletonsOfSdk(@Nullable Project project, Component ownerComponent, String skeletonsPath, - @Nullable Ref<Boolean> migrationFlag, @NotNull Sdk sdk) throws InvalidSdkException { final Map<String, List<String>> errors = new TreeMap<String, List<String>>(); @@ -137,7 +127,7 @@ public class PySkeletonRefresher { changeGeneratingSkeletons(1); try { - List<String> sdkErrors = refresher.regenerateSkeletons(checker, migrationFlag); + List<String> sdkErrors = refresher.regenerateSkeletons(checker); if (sdkErrors.size() > 0) { String sdkName = sdk.getName(); List<String> knownErrors = errors.get(sdkName); @@ -286,8 +276,7 @@ public class PySkeletonRefresher { return mySkeletonsPath; } - public List<String> regenerateSkeletons(@Nullable SkeletonVersionChecker cachedChecker, - @Nullable Ref<Boolean> migrationFlag) throws InvalidSdkException { + public List<String> regenerateSkeletons(@Nullable SkeletonVersionChecker cachedChecker) throws InvalidSdkException { final List<String> errorList = new SmartList<String>(); final String homePath = mySdk.getHomePath(); final String skeletonsPath = getSkeletonsPath(); @@ -299,14 +288,13 @@ public class PySkeletonRefresher { final String readablePath = FileUtil.getLocationRelativeToUserHome(homePath); mySkeletonsGenerator.prepare(); - myBlacklist = loadBlacklist(); indicate(PyBundle.message("sdk.gen.querying.$0", readablePath)); // get generator version and binary libs list in one go - final PySkeletonGenerator.ListBinariesResult binaries = - mySkeletonsGenerator.listBinaries(mySdk, calculateExtraSysPath(mySdk, getSkeletonsPath())); + final String extraSysPath = calculateExtraSysPath(mySdk, getSkeletonsPath()); + final PySkeletonGenerator.ListBinariesResult binaries = mySkeletonsGenerator.listBinaries(mySdk, extraSysPath); myGeneratorVersion = binaries.generatorVersion; myPregeneratedSkeletons = findPregeneratedSkeletons(); @@ -325,77 +313,19 @@ public class PySkeletonRefresher { final SkeletonHeader oldHeader = readSkeletonHeader(builtinsFile); final boolean oldOrNonExisting = oldHeader == null || oldHeader.getVersion() == 0; - if (migrationFlag != null && !migrationFlag.get() && oldOrNonExisting) { - migrationFlag.set(true); - Notifications.Bus.notify( - new Notification( - PythonSdkType.SKELETONS_TOPIC, PyBundle.message("sdk.gen.notify.converting.old.skels"), - PyBundle.message("sdk.gen.notify.converting.text"), - NotificationType.INFORMATION - ) - ); - } - if (myPregeneratedSkeletons != null && oldOrNonExisting) { - indicate("Unpacking pregenerated skeletons..."); - try { - final VirtualFile jar = JarFileSystem.getInstance().getVirtualFileForJar(myPregeneratedSkeletons); - if (jar != null) { - ZipUtil.extract(new File(jar.getPath()), - new File(getSkeletonsPath()), null); - } - } - catch (IOException e) { - LOG.info("Error unpacking pregenerated skeletons", e); - } + unpackPreGeneratedSkeletons(); } if (oldOrNonExisting) { - final Sdk base = PythonSdkType.getInstance().getVirtualEnvBaseSdk(mySdk); - if (base != null) { - indicate("Copying base SDK skeletons for virtualenv..."); - final String baseSkeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), base.getHomePath()); - final PySkeletonGenerator.ListBinariesResult baseBinaries = - mySkeletonsGenerator.listBinaries(base, calculateExtraSysPath(base, baseSkeletonsPath)); - for (Map.Entry<String, PyBinaryItem> entry : binaries.modules.entrySet()) { - final String module = entry.getKey(); - final PyBinaryItem binary = entry.getValue(); - final PyBinaryItem baseBinary = baseBinaries.modules.get(module); - final File fromFile = getSkeleton(module, baseSkeletonsPath); - if (baseBinaries.modules.containsKey(module) && - fromFile.exists() && - binary.length() == baseBinary.length()) { // Weak binary modules equality check - final File toFile = fromFile.isDirectory() ? - getPackageSkeleton(module, skeletonsPath) : - getModuleSkeleton(module, skeletonsPath); - try { - FileUtil.copy(fromFile, toFile); - } - catch (IOException e) { - LOG.info("Error copying base virtualenv SDK skeleton for " + module, e); - } - } - } - } + copyBaseSdkSkeletonsToVirtualEnv(skeletonsPath, binaries); } - final SkeletonHeader newHeader = readSkeletonHeader(builtinsFile); - final boolean mustUpdateBuiltins = myPregeneratedSkeletons == null && - (newHeader == null || newHeader.getVersion() < myVersionChecker.getBuiltinVersion()); - if (mustUpdateBuiltins) { - indicate(PyBundle.message("sdk.gen.updating.builtins.$0", readablePath)); - mySkeletonsGenerator.generateBuiltinSkeletons(mySdk); - if (myProject != null) { - PythonSdkPathCache.getInstance(myProject, mySdk).clearBuiltins(); - } - } + final boolean builtinsUpdated = updateSkeletonsForBuiltins(readablePath, builtinsFile); if (!binaries.modules.isEmpty()) { - indicate(PyBundle.message("sdk.gen.updating.$0", readablePath)); - - List<UpdateResult> updateErrors = updateOrCreateSkeletons(binaries.modules); //Skeletons regeneration - + final List<UpdateResult> updateErrors = updateOrCreateSkeletons(binaries.modules); if (updateErrors.size() > 0) { indicateMinor(BLACKLIST_FILE_NAME); for (UpdateResult error : updateErrors) { @@ -410,7 +340,6 @@ public class PySkeletonRefresher { } indicate(PyBundle.message("sdk.gen.reloading")); - mySkeletonsGenerator.refreshGeneratedSkeletons(); if (!oldOrNonExisting) { @@ -419,14 +348,15 @@ public class PySkeletonRefresher { } if (PySdkUtil.isRemote(mySdk)) { try { - ((PyPackageManagerImpl)PyPackageManager.getInstance(mySdk)).loadPackages(); + // Force loading packages + PyPackageManager.getInstance(mySdk).getPackages(false); } catch (PyExternalProcessException e) { // ignore - already logged } } - if ((mustUpdateBuiltins || PySdkUtil.isRemote(mySdk)) && myProject != null) { + if ((builtinsUpdated || PySdkUtil.isRemote(mySdk)) && myProject != null) { ApplicationManager.getApplication().invokeLater(new Runnable() { @Override public void run() { @@ -438,6 +368,64 @@ public class PySkeletonRefresher { return errorList; } + private boolean updateSkeletonsForBuiltins(String readablePath, File builtinsFile) throws InvalidSdkException { + final SkeletonHeader newHeader = readSkeletonHeader(builtinsFile); + final boolean mustUpdateBuiltins = myPregeneratedSkeletons == null && + (newHeader == null || newHeader.getVersion() < myVersionChecker.getBuiltinVersion()); + if (mustUpdateBuiltins) { + indicate(PyBundle.message("sdk.gen.updating.builtins.$0", readablePath)); + mySkeletonsGenerator.generateBuiltinSkeletons(mySdk); + if (myProject != null) { + PythonSdkPathCache.getInstance(myProject, mySdk).clearBuiltins(); + } + } + return mustUpdateBuiltins; + } + + private void copyBaseSdkSkeletonsToVirtualEnv(String skeletonsPath, PySkeletonGenerator.ListBinariesResult binaries) + throws InvalidSdkException { + final Sdk base = PythonSdkType.getInstance().getVirtualEnvBaseSdk(mySdk); + if (base != null) { + indicate("Copying base SDK skeletons for virtualenv..."); + final String baseSkeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), base.getHomePath()); + final PySkeletonGenerator.ListBinariesResult baseBinaries = + mySkeletonsGenerator.listBinaries(base, calculateExtraSysPath(base, baseSkeletonsPath)); + for (Map.Entry<String, PyBinaryItem> entry : binaries.modules.entrySet()) { + final String module = entry.getKey(); + final PyBinaryItem binary = entry.getValue(); + final PyBinaryItem baseBinary = baseBinaries.modules.get(module); + final File fromFile = getSkeleton(module, baseSkeletonsPath); + if (baseBinaries.modules.containsKey(module) && + fromFile.exists() && + binary.length() == baseBinary.length()) { // Weak binary modules equality check + final File toFile = fromFile.isDirectory() ? + getPackageSkeleton(module, skeletonsPath) : + getModuleSkeleton(module, skeletonsPath); + try { + FileUtil.copy(fromFile, toFile); + } + catch (IOException e) { + LOG.info("Error copying base virtualenv SDK skeleton for " + module, e); + } + } + } + } + } + + private void unpackPreGeneratedSkeletons() throws InvalidSdkException { + indicate("Unpacking pregenerated skeletons..."); + try { + final VirtualFile jar = JarFileSystem.getInstance().getVirtualFileForJar(myPregeneratedSkeletons); + if (jar != null) { + ZipUtil.extract(new File(jar.getPath()), + new File(getSkeletonsPath()), null); + } + } + catch (IOException e) { + LOG.info("Error unpacking pregenerated skeletons", e); + } + } + @Nullable public static SkeletonHeader readSkeletonHeader(@NotNull File file) { try { @@ -835,7 +823,7 @@ public class PySkeletonRefresher { return null; } LOG.info("Pregenerated skeletons root is " + root); - final String versionString = mySdk.getVersionString(); + @NonNls final String versionString = mySdk.getVersionString(); if (versionString == null) { return null; } |