Fixed an issue when configuration file options partly ignored when ``…

…--project-conf`` // Resolve #3034 (#3055)

* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` // Resolve #3034

* Py2 compatible makedirs

* Fix circle dependency

* Fix broken import in test examples

* Fix history

* Remove YAPF markers

* PyLint fix

* Fix invalid project conf path

* Move PIO Core to the root on Windows, issue with long CPPPATHs

* Respect global PLATFORMIO_BUILD_CACHE_DIR env var

* Fix Appveyor paths

* Minor changes

.appveyor.yml

@@ -6,14 +6,15 @@ platform:
 environment:
   matrix:
     - TOXENV: "py27"
-      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build}
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build}
 
     - TOXENV: "py36"
-      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build}
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build}
 
 install:
   - cmd: git submodule update --init --recursive
   - cmd: SET PATH=C:\MinGW\bin;%PATH%
+  - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio
   - cmd: pip install --force-reinstall tox
 
 test_script:

.pylintrc

@@ -11,4 +11,5 @@ disable=
     too-few-public-methods,
     useless-object-inheritance,
     useless-import-alias,
-    fixme
+    fixme,
+    bad-option-value

HISTORY.rst

@@ -23,6 +23,7 @@ PlatformIO Core 4.0
 * Added ``--no-ansi`` flag for `PIO Core <http://docs.platformio.org/page/userguide/index.html>`__ to disable ANSI control characters
 * Fixed an issue with project generator for `CLion IDE <http://docs.platformio.org/page/ide/clion.html>`__ when 2 environments were used (`issue #2824 <https://github.com/platformio/platformio-core/issues/2824>`_)
 * Fixed default PIO Unified Debugger configuration for `J-Link probe <http://docs.platformio.org/page/plus/debug-tools/jlink.html>`__
+* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 <https://github.com/platformio/platformio-core/issues/3034>`_)
 
 4.0.3 (2019-08-30)
 ~~~~~~~~~~~~~~~~~~

platformio/__main__.py

@@ -70,7 +70,7 @@ def configure():
     # https://urllib3.readthedocs.org
     # /en/latest/security.html#insecureplatformwarning
     try:
-        import urllib3
+        import urllib3  # pylint: disable=import-outside-toplevel
 
         urllib3.disable_warnings()
     except (AttributeError, ImportError):

platformio/app.py

@@ -17,29 +17,19 @@
 import os
 import uuid
 from os import environ, getenv, listdir, remove
-from os.path import abspath, dirname, expanduser, isdir, isfile, join
+from os.path import abspath, dirname, isdir, isfile, join
 from time import time
 
 import requests
 
 from platformio import exception, fs, lockfile
 from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
 from platformio.proc import is_ci
-from platformio.project.helpers import get_project_cache_dir, get_project_core_dir
-
-
-def get_default_projects_dir():
-    docs_dir = join(expanduser("~"), "Documents")
-    try:
-        assert WINDOWS
-        import ctypes.wintypes
-
-        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
-        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
-        docs_dir = buf.value
-    except:  # pylint: disable=bare-except
-        pass
-    return join(docs_dir, "PlatformIO", "Projects")
+from platformio.project.helpers import (
+    get_default_projects_dir,
+    get_project_cache_dir,
+    get_project_core_dir,
+)
 
 
 def projects_dir_validate(projects_dir):
@@ -88,7 +78,12 @@ def projects_dir_validate(projects_dir):
     },
 }
 
-SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None}
+SESSION_VARS = {
+    "command_ctx": None,
+    "force_option": False,
+    "caller_id": None,
+    "custom_project_conf": None,
+}
 
 
 class State(object):
@@ -415,6 +410,6 @@ def get_cid():
         uid = uuid.getnode()
     cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
     cid = str(cid)
-    if WINDOWS or os.getuid() > 0:  # yapf: disable pylint: disable=no-member
+    if WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
         set_state_item("cid", cid)
     return cid

platformio/builder/main.py

@@ -31,7 +31,7 @@
 from platformio.compat import PY2, dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
-from platformio.project import helpers as project_helpers
+from platformio.project.helpers import get_project_dir
 
 AllowSubstExceptions(NameError)
 
@@ -44,7 +44,7 @@
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
     ("UPLOAD_PORT",),
-)  # yapf: disable
+)
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
@@ -67,26 +67,10 @@
     # Propagating External Environment
     ENV=environ,
     UNIX_TIME=int(time()),
-    PROJECT_DIR=project_helpers.get_project_dir(),
-    PROJECTCORE_DIR=project_helpers.get_project_core_dir(),
-    PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(),
-    PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(),
-    PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(),
-    PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(),
-    PROJECTSRC_DIR=project_helpers.get_project_src_dir(),
-    PROJECTTEST_DIR=project_helpers.get_project_test_dir(),
-    PROJECTDATA_DIR=project_helpers.get_project_data_dir(),
-    PROJECTBUILD_DIR=project_helpers.get_project_build_dir(),
-    BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"),
-    BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
-    BUILDSRC_DIR=join("$BUILD_DIR", "src"),
-    BUILDTEST_DIR=join("$BUILD_DIR", "test"),
+    BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
+    BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
+    BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
     LIBPATH=["$BUILD_DIR"],
-    LIBSOURCE_DIRS=[
-        project_helpers.get_project_lib_dir(),
-        join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
-        project_helpers.get_project_global_lib_dir(),
-    ],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
     PYTHONEXE=get_pythonexe_path(),
@@ -110,10 +94,33 @@
     }
 )
 
-if env.subst("$BUILDCACHE_DIR"):
-    if not isdir(env.subst("$BUILDCACHE_DIR")):
-        makedirs(env.subst("$BUILDCACHE_DIR"))
-    env.CacheDir("$BUILDCACHE_DIR")
+# Setup project optional directories
+config = env.GetProjectConfig()
+env.Replace(
+    PROJECT_DIR=get_project_dir(),
+    PROJECT_CORE_DIR=config.get_optional_dir("core"),
+    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
+    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
+    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
+    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
+    PROJECT_SRC_DIR=config.get_optional_dir("src"),
+    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
+    PROJECT_TEST_DIR=config.get_optional_dir("test"),
+    PROJECT_DATA_DIR=config.get_optional_dir("data"),
+    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
+    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
+    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    LIBSOURCE_DIRS=[
+        config.get_optional_dir("lib"),
+        join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
+        config.get_optional_dir("globallib"),
+    ],
+)
+
+if env.subst("$BUILD_CACHE_DIR"):
+    if not isdir(env.subst("$BUILD_CACHE_DIR")):
+        makedirs(env.subst("$BUILD_CACHE_DIR"))
+    env.CacheDir("$BUILD_CACHE_DIR")
 
 if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access

platformio/builder/tools/pioide.py

@@ -54,7 +54,7 @@ def _dump_includes(env, projenv):
     if unity_dir:
         includes.append(unity_dir)
 
-    includes.extend([env.subst("$PROJECTINCLUDE_DIR"), env.subst("$PROJECTSRC_DIR")])
+    includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")])
 
     # remove duplicates
     result = []

platformio/builder/tools/piolib.py

@@ -601,8 +601,7 @@ def _process_mbed_lib_confs(self):
             mbed_config_path = join(self.env.subst(p), "mbed_config.h")
             if isfile(mbed_config_path):
                 break
-            else:
-                mbed_config_path = None
+            mbed_config_path = None
         if not mbed_config_path:
             return None
 
@@ -821,16 +820,16 @@ def __init__(self, env, *args, **kwargs):
 
     @property
     def include_dir(self):
-        include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
+        include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         return include_dir if isdir(include_dir) else None
 
     @property
     def src_dir(self):
-        return self.env.subst("$PROJECTSRC_DIR")
+        return self.env.subst("$PROJECT_SRC_DIR")
 
     def get_include_dirs(self):
         include_dirs = []
-        project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
+        project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         if isdir(project_include_dir):
             include_dirs.append(project_include_dir)
         for include_dir in LibBuilderBase.get_include_dirs(self):
@@ -845,9 +844,9 @@ def get_search_files(self):
         if "__test" in COMMAND_LINE_TARGETS:
             items.extend(
                 [
-                    join("$PROJECTTEST_DIR", item)
+                    join("$PROJECT_TEST_DIR", item)
                     for item in self.env.MatchSourceFiles(
-                        "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER"
+                        "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
                     )
                 ]
             )
@@ -896,7 +895,7 @@ def _is_builtin(uri):
                 not_found_uri.append(uri)
 
         did_install = False
-        lm = LibraryManager(self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV")))
+        lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")))
         for uri in not_found_uri:
             try:
                 lm.install(uri)

platformio/builder/tools/piomisc.py

@@ -117,7 +117,7 @@ def _join_multiline_strings(self, contents):
                     stropen = True
                     newlines.append(line[:-1])
                     continue
-                elif stropen:
+                if stropen:
                     newlines[len(newlines) - 1] += line[:-1]
                     continue
             elif stropen and line.endswith(('",', '";')):
@@ -199,7 +199,7 @@ def append_prototypes(self, contents):
 
 
 def ConvertInoToCpp(env):
-    src_dir = glob_escape(env.subst("$PROJECTSRC_DIR"))
+    src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR"))
     ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))
     if not ino_nodes:
         return
@@ -256,7 +256,7 @@ def _lookup_in_ldpath(script):
         if f == "-T":
             script_in_next = True
             continue
-        elif script_in_next:
+        if script_in_next:
             script_in_next = False
             raw_script = f
         elif f.startswith("-Wl,-T"):
@@ -309,7 +309,7 @@ def PioClean(env, clean_dir):
     env.Exit(0)
 
 
-def ProcessDebug(env):
+def ConfigureDebugTarget(env):
     if not env.subst("$PIODEBUGFLAGS"):
         env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"])
     env.Append(
@@ -322,7 +322,7 @@ def ProcessDebug(env):
     env.Append(BUILD_UNFLAGS=unflags)
 
 
-def ProcessTest(env):
+def ConfigureTestTarget(env):
     env.Append(
         CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
         CPPPATH=[join("$BUILD_DIR", "UnityTestLib")],
@@ -361,7 +361,7 @@ def generate(env):
     env.AddMethod(GetActualLDScript)
     env.AddMethod(VerboseAction)
     env.AddMethod(PioClean)
-    env.AddMethod(ProcessDebug)
-    env.AddMethod(ProcessTest)
+    env.AddMethod(ConfigureDebugTarget)
+    env.AddMethod(ConfigureTestTarget)
     env.AddMethod(GetExtraScripts)
     return env

platformio/builder/tools/platformio.py

@@ -67,15 +67,17 @@ def _build_project_deps(env):
     is_test = "__test" in COMMAND_LINE_TARGETS
     if is_test:
         projenv.BuildSources(
-            "$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER"
+            "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
         )
     if not is_test or env.GetProjectOption("test_build_project_src", False):
-        projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER"))
+        projenv.BuildSources(
+            "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
+        )
 
     if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR")
+            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
         )
         env.Exit(1)
 
@@ -102,7 +104,7 @@ def _append_pio_macros():
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
     if "debug" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "debug":
-        env.ProcessDebug()
+        env.ConfigureDebugTarget()
 
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
@@ -121,7 +123,7 @@ def _append_pio_macros():
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
     if "__test" in COMMAND_LINE_TARGETS:
-        env.ProcessTest()
+        env.ConfigureTestTarget()
 
     # build project with dependencies
     _build_project_deps(env)

platformio/check/tools/cppcheck.py

@@ -19,7 +19,6 @@
 from platformio.check.defect import DefectItem
 from platformio.check.tools.base import CheckToolBase
 from platformio.managers.core import get_core_package_dir
-from platformio.project.helpers import get_project_core_dir
 
 
 class CppcheckCheckTool(CheckToolBase):
@@ -110,7 +109,7 @@ def configure_command(self):
         cmd.append("--file-list=%s" % self._generate_src_file())
         cmd.append("--includes-file=%s" % self._generate_inc_file())
 
-        core_dir = get_project_core_dir()
+        core_dir = self.config.get_optional_dir("core")
         cmd.append("--suppress=*:%s*" % core_dir)
         cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir)