Merge pull request nickderobertis#54 from nickderobertis/re-forecast-bug

Auto-adjust config based on availability of data items

.github/workflows/automerge.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
 
     steps:
       - name: Dump GitHub context
@@ -55,10 +55,10 @@ jobs:
     needs: test
     steps:
       - uses: actions/checkout@v1
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.8
       - name: Check if maintainer
         env:
           GITHUB_PR_USER: ${{ github.actor }}
@@ -100,7 +100,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - uses: actions/checkout@v1
         with:
@@ -127,7 +127,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - uses: actions/checkout@v1
         with:

.github/workflows/docs.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v1
@@ -65,7 +65,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v1

.github/workflows/package.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
 
     steps:
     - name: Dump GitHub context
@@ -75,7 +75,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - uses: actions/checkout@v1
       - name: Collect TODO
@@ -104,7 +104,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - uses: actions/checkout@v1
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/template-update.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v2

Pipfile

@@ -6,7 +6,7 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-sphinx = "==2.4.3"
+sphinx = "*"
 sphinx-autobuild = "*"
 sphinx-autodoc-typehints = "*"
 sphinxcontrib-fulltoc = "*"
@@ -31,6 +31,7 @@ fbprophet = "*"
 tqdm = "*"
 statsmodels = "*"
 pillow = "*"
+pydantic = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"

conf.py

@@ -20,7 +20,7 @@
 ]
 
 # Package version in the format (major, minor, release)
-PACKAGE_VERSION_TUPLE = (0, 6, 4)
+PACKAGE_VERSION_TUPLE = (0, 7, 0)
 
 # Short description of the package
 PACKAGE_SHORT_DESCRIPTION = 'Python package for working with financial statement data'
@@ -68,6 +68,7 @@
     'matplotlib',
     'tqdm',
     'statsmodels',
+    'pydantic',
 ]
 
 # Add any third party packages you use in requirements for optional features of your package here

finstmt/bs/config.py

@@ -158,7 +158,7 @@
     ),
     ItemConfig(
         'gross_ppe',
-        'Grosss Property, Plant & Equipment',
+        'Gross Property, Plant & Equipment',
         extract_names=[
             'gross ppe',
             'gross property plant equipment',

finstmt/check.py

@@ -0,0 +1,11 @@
+import pandas as pd
+
+
+def item_series_is_empty(s: pd.Series) -> bool:
+    if s.sum() != 0:
+        return False
+    for value in s:
+        if value != 0:
+            return False
+
+    return True

finstmt/combined/statements.py

@@ -1,25 +1,33 @@
 import operator
 from copy import deepcopy
 from dataclasses import dataclass, field
-from typing import Dict, List, Tuple, Callable
+from typing import Dict, List, Tuple, Callable, Set
 
 import pandas as pd
 from sympy import Indexed
 
 from finstmt import BalanceSheets, IncomeStatements
+from finstmt.check import item_series_is_empty
 from finstmt.config_manage.statements import StatementsConfigManager
 from finstmt.exc import MismatchingDatesException
 from finstmt.findata.statementsbase import FinStatementsBase
 from finstmt.forecast.config import ForecastConfig
 from finstmt.forecast.main import Forecast
 from finstmt.items.config import ItemConfig
+from finstmt.logger import logger
 
 
 @dataclass
 class FinancialStatements:
     """
     Main class that holds all the financial statements.
 
+    :param auto_adjust_config: Whether to automatically adjust the configuration based
+        on the loaded data. Currently will turn forecasting off for items not in the data,
+        and turn forecasting on for items normally calculated off those which are
+        not in the data. For example, if gross_ppe is missing then will start forecasting
+        net_ppe instead
+
     Examples:
         >>> bs_path = r'WMT Balance Sheet.xlsx'
         >>> inc_path = r'WMT Income Statement.xlsx'
@@ -32,6 +40,7 @@ class FinancialStatements:
     income_statements: IncomeStatements
     balance_sheets: BalanceSheets
     calculate: bool = True
+    auto_adjust_config: bool = True
 
 
     def __post_init__(self):
@@ -41,7 +50,7 @@ def __post_init__(self):
 
         if self.calculate:
             resolver = StatementsResolver(self)
-            new_stmts = resolver.to_statements()
+            new_stmts = resolver.to_statements(auto_adjust_config=self.auto_adjust_config)
             self.income_statements = new_stmts.income_statements
             self.balance_sheets = new_stmts.balance_sheets
             self._create_config_from_statements()
@@ -50,7 +59,55 @@ def _create_config_from_statements(self):
         config_dict = {}
         config_dict['income_statements'] = self.income_statements.config
         config_dict['balance_sheets'] = self.balance_sheets.config
-        self.config = StatementsConfigManager(config_dict)
+        self.config = StatementsConfigManager(config_managers=config_dict)
+        if self.auto_adjust_config:
+            self._adjust_config_based_on_data()
+
+    def _adjust_config_based_on_data(self):
+        for item in self.config.items:
+            if self.item_is_empty(item.key):
+                if self.config.get(item.key).forecast_config.plug:
+                    # It is OK for plug items to be empty, won't affect the forecast
+                    continue
+
+                # Useless to make forecasts on empty items
+                logger.debug(f'Setting {item.key} to not forecast as it is empty')
+                item.forecast_config.make_forecast = False
+                # But this may mean another item should be forecasted instead.
+                # E.g. normally net_ppe is calculated from gross_ppe and dep,
+                # so it is not forecasted. But if gross_ppe is missing from
+                # the data, then net_ppe should be forecasted directly.
+
+                # So first, get the equations involving this item to determine
+                # what other items are related to this one
+                relevant_eqs = self.config.eqs_involving(item.key)
+                relevant_keys: Set[str] = {item.key}
+                for eq in relevant_eqs:
+                    relevant_keys.add(self.config._expr_to_keys(eq.lhs)[0])
+                    relevant_keys.update(set(self.config._expr_to_keys(eq.rhs)))
+                relevant_keys.remove(item.key)
+                for key in relevant_keys:
+                    if self.item_is_empty(key):
+                        continue
+                    conf = self.config.get(key)
+                    if conf.expr_str is None:
+                        # Not a calculated item, so it doesn't make sense to turn forecasting on
+                        continue
+
+                    # Check to make sure that all components of the calculated item are also empty
+                    expr = self.config.expr_for(key)
+                    component_keys = self.config._expr_to_keys(expr)
+                    all_component_items_are_empty = True
+                    for c_key in component_keys:
+                        if not self.item_is_empty(c_key):
+                            all_component_items_are_empty = False
+                    if not all_component_items_are_empty:
+                        continue
+                    # Now this is a calculated item which is non-empty, and all the components of the
+                    # calculated are empty, so we need to forecast this item instead
+                    logger.debug(f'Setting {conf.key} to forecast as it is a calculated item which is not empty '
+                                 f'and yet none of the components have data')
+                    conf.forecast_config.make_forecast = True
 
     def change(self, data_key: str) -> pd.Series:
         """
@@ -71,6 +128,16 @@ def lag(self, data_key: str, num_lags: int) -> pd.Series:
         series = getattr(self, data_key)
         return series.shift(num_lags)
 
+    def item_is_empty(self, data_key: str) -> bool:
+        """
+        Whether the passed item has no data
+
+        :param data_key: key of variable, how it would be accessed with FinancialStatements.data_key
+        :return:
+        """
+        series = getattr(self, data_key)
+        return item_series_is_empty(series)
+
     def _repr_html_(self):
         return f"""
         <h2>Income Statement</h2>

finstmt/config_manage/base.py

@@ -1,6 +1,6 @@
-from typing import Dict, Any
+from typing import Dict, Any, List
 
-from sympy import symbols, IndexedBase, Idx, Expr, sympify
+from sympy import symbols, IndexedBase, Idx, Expr, sympify, Eq
 
 from finstmt.exc import NotACalculatedItemException
 from finstmt.items.config import ItemConfig
@@ -28,6 +28,13 @@ def sympy_namespace(self) -> Dict[str, IndexedBase]:
         """
         raise NotImplementedError
 
+    @property
+    def items(self) -> List[ItemConfig]:
+        """
+        All the configuration items
+        """
+        raise NotImplementedError
+
     def get_value(self, item_key: str, config_key: str) -> Any:
         """
         Get a particular configuration for a particular item
@@ -50,6 +57,37 @@ def expr_for(self, item_key: str) -> Expr:
         expr = sympify(config.expr_str, locals=self.sympy_namespace)
         return expr
 
+    def eqs_involving(self, item_key: str) -> List[Eq]:
+        ns = self.sympy_namespace
+        item_sym = ns[item_key]
+        t = ns['t']
+        item_t = item_sym[t]
+        eqs: List[Eq] = []
+        for config in self.items:
+            if config.expr_str is None:
+                continue
+            rhs = self.expr_for(config.key)
+            if item_t in rhs.free_symbols:
+                this_item_sym = ns[config.key]
+                this_item_sym_t = this_item_sym[t]
+                eq = Eq(this_item_sym_t, rhs)
+                eqs.append(eq)
+
+        return eqs
+
+    def _expr_to_keys(self, expr: Expr) -> List[str]:
+        ns = self.sympy_namespace
+        t = ns['t']
+        syms = expr.free_symbols
+        keys: List[str] = []
+        for key, key_expr in ns.items():
+            if key == 't':
+                continue
+            key_expr_t = key_expr[t]
+            if key_expr_t in syms:
+                keys.append(key)
+        return keys
+
     def eq_subs_dict(self, values_dict: Dict[str, float], t_offset: int = 0) -> Dict[IndexedBase, float]:
         out_dict = {}
         t = self.sympy_namespace['t']

finstmt/config_manage/data.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from pydantic.dataclasses import dataclass
 from typing import Dict, List
 
 from sympy import symbols, IndexedBase, Idx
@@ -61,6 +61,10 @@ def sympy_namespace(self) -> Dict[str, IndexedBase]:
     def keys(self) -> List[str]:
         return list(self.config_dict.keys())
 
+    @property
+    def items(self) -> List[ItemConfig]:
+        return self.configs
+
 
 def _key_pct_of_key(base_key: str, pct_of_key: str) -> str:
     return f'{base_key}_pct_{pct_of_key}'

finstmt/config_manage/global_.py

@@ -12,7 +12,7 @@
 inc_stmt_config_mgr = StatementConfigManager({pd.Timestamp(datetime.datetime.today()): inc_data_config_mgr})
 bs_data_config_mgr = DataConfigManager(BALANCE_SHEET_INPUT_ITEMS)
 bs_stmt_config_mgr = StatementConfigManager({pd.Timestamp(datetime.datetime.today()): bs_data_config_mgr})
-CONFIG = StatementsConfigManager({
+CONFIG = StatementsConfigManager(config_managers={
     'income_statements': inc_stmt_config_mgr,
     'balance_sheets': bs_stmt_config_mgr
 })

finstmt/config_manage/statement.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from pydantic.dataclasses import dataclass
 from typing import Dict, Tuple, List
 
 from sympy import IndexedBase

finstmt/config_manage/statements.py

@@ -1,6 +1,8 @@
-from dataclasses import dataclass
+import json
+from dataclasses import asdict
 from typing import Dict, Tuple, Sequence, Union, Any, List
 
+from pydantic.dataclasses import dataclass
 from sympy import IndexedBase
 
 from finstmt.config_manage.base import ConfigManagerBase
@@ -101,3 +103,12 @@ def items(self) -> List[ItemConfig]:
                     continue
                 all_items.append(item)
         return all_items
+
+    def dict(self) -> dict:
+        item_data: Dict[str, dict] = {}
+        for item in self.items:
+            item_data[item.key] = asdict(item)
+        return item_data
+
+    def json(self, **kwargs) -> str:
+        return json.dumps(self.dict(), **kwargs)