Merge pull request #172 from JECSand/development

JECSand · web-flow · commit 2f2cc6c09e55 · 2023-12-17T01:54:07.000-06:00
Development
diff --git a/CHANGES b/CHANGES
@@ -59,3 +59,6 @@
 1.18  12/09/2023 -- Merged in branch from shaunpatterson to fix #164.
 1.19  12/12/2023 -- Refactored session management system to handle cookie and crumbs better.
 1.19  12/12/2023 -- Added fixes for #167, #166, #160.
+1.20  12/16/2023 -- Merged in pull request #171 from bjosun.
+1.20  12/17/2023 -- Added optional flat format output param on YahooFinancial class.
+1.20  12/17/2023 -- Added get_insight() and get_recommendations() methods.
diff --git a/README.rst b/README.rst
@@ -16,16 +16,71 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit
 .. image:: https://static.pepy.tech/badge/yahoofinancials/week
     :target: https://pepy.tech/project/yahoofinancials
 
-Current Version: v1.19
+Current Version: v1.20
 
-Version Released: 12/12/2023
+Version Released: 12/17/2023
 
 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues
 
 Overview
 --------
 A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance.
 
+- New analytic methods in v1.20:
+    - get_insights()
+        - returns data for:
+            - 'instrumentInfo'
+            - 'companySnapshot'
+            - 'recommendation'
+            - 'sigDevs'
+            - 'secReports'
+    - get_recommendations()
+
+- Example:
+
+.. code-block:: python
+
+    print(YahooFinancials('C').get_recommendations())
+
+- Example Output:
+
+.. code-block:: javascript
+
+    {
+        "C": [
+            {
+                "recommendedSymbols": [
+                    {
+                        "score": 0.239602,
+                        "symbol": "BAC"
+                    },
+                    {
+                    "score": 0.225134,
+                    "symbol": "JPM"
+                    },
+                    {
+                    "score": 0.167669,
+                    "symbol": "WFC"
+                    },
+                    {
+                    "score": 0.145864,
+                    "symbol": "GS"
+                    },
+                    {
+                    "score": 0.134071,
+                    "symbol": "F"
+                    }
+                ],
+                "symbol": "C"
+            }
+        ]
+    }
+
+- As of Version 1.20, YahooFinancials supports a new optional parameter called flat_format.
+    - When `YahooFinancials(flat_format=True)`, financial statement data will return in a dict instead of a list. The keys of the dict will be the reporting dates.
+    - Default is False, to ensure backwards compatibility.
+
+
 - As of Version 1.9, YahooFinancials supports optional parameters for asynchronous execution, proxies, and international requests.
 
 .. code-block:: python
@@ -41,10 +96,6 @@ A powerful financial data module used for pulling both fundamental and technical
     balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance')
     print(balance_sheet_data_qt)
 
-- New methods in Version 1.13:
-    - get_esg_score_data()
-
-
 Installation
 -------------
 - yahoofinancials runs on Python 3.7, 3.8, 3.9, 3.10, 3.11, and 3.12
@@ -91,7 +142,7 @@ Module Methods
 --------------
 - The financial data from all methods is returned as JSON.
 - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string.
-- YahooFinancials works with Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 and runs on all operating systems. (Windows, Mac, Linux).
+- YahooFinancials works with Python 3.7, 3.8, 3.9, 3.10, 3.11 and 3.12 and runs on all operating systems. (Windows, Mac, Linux).
 
 Featured Methods
 ^^^^^^^^^^^^^^^^
@@ -134,6 +185,7 @@ Additional Module Methods
 - get_cost_of_revenue()
 - get_income_before_tax()
 - get_income_tax_expense()
+- get_esg_score_data()
 - get_gross_profit()
 - get_net_income_from_continuing_ops()
 - get_research_and_development()
diff --git a/setup.py b/setup.py
@@ -10,11 +10,11 @@
 
 setup(
     name='yahoofinancials',
-    version='1.19',
+    version='1.20',
     description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance',
     long_description=long_description,
     url='https://github.com/JECSand/yahoofinancials',
-    download_url='https://github.com/JECSand/yahoofinancials/archive/1.19.tar.gz',
+    download_url='https://github.com/JECSand/yahoofinancials/archive/1.20.tar.gz',
     author='Connor Sanders',
     author_email='connor@exceleri.com',
     license='MIT',
diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py
@@ -1,7 +1,7 @@
-# YahooFinancials Unit Tests v1.19
-# Version Released: 12/12/2023
+# YahooFinancials Unit Tests v1.20
+# Version Released: 12/17/2023
 # Author: Connor Sanders
-# Tested on Python 3.7, 3.8, 3.9, 3.10, and 3.11
+# Tested on Python 3.7, 3.8, 3.9, 3.10, 3.11, and 3.12
 # Copyright (c) 2023 Connor Sanders <jecsand@pm.me>
 # MIT License
 
@@ -42,6 +42,8 @@ def setUp(self):
         self.test_yf_treasuries_multi = yf(us_treasuries)
         self.test_yf_currencies = yf(currencies)
         self.test_yf_concurrent = yf(stocks, concurrent=True)
+        self.test_yf_stock_flat = yf('C', flat_format=True)
+        self.test_yf_stock_analytic = yf('WFC')
 
     # Fundamentals Test
     def test_yf_fundamentals(self):
@@ -103,6 +105,35 @@ def test_yf_concurrency(self):
         result = check_fundamental(multi_all_statement_data_qt, 'all')
         self.assertEqual(result, True)
 
+    # Fundamentals in Flat Format Test
+    def test_yf_fundamentals_flat(self):
+        # Single stock test
+        single_all_statement_data_qt = self.test_yf_stock_flat.get_financial_stmts('quarterly',
+                                                                                     ['income', 'cash', 'balance'])
+        if ((isinstance(single_all_statement_data_qt.get("incomeStatementHistoryQuarterly").get("C"), dict) and
+                isinstance(single_all_statement_data_qt.get("balanceSheetHistoryQuarterly").get("C"), dict)) and
+                isinstance(single_all_statement_data_qt.get("cashflowStatementHistoryQuarterly").get("C"), dict)):
+            self.assertEqual(True, True)
+        else:
+            self.assertEqual(False, True)
+
+    # Analytic Methods Test
+    def test_yf_analytic_methods(self):
+
+        # Get Insights
+        out = self.test_yf_stock_analytic.get_insights()
+        if out.get("WFC").get("instrumentInfo").get("technicalEvents").get("sector") == "Financial Services":
+            self.assertEqual(True, True)
+        else:
+            self.assertEqual(False, True)
+
+        # Get Recommendations
+        out = self.test_yf_stock_analytic.get_recommendations()
+        if isinstance(out.get("WFC"), list):
+            self.assertEqual(True, True)
+        else:
+            self.assertEqual(False, True)
+
     # Extra Module Methods Test
     def test_yf_module_methods(self):
 
diff --git a/yahoofinancials/cache.py b/yahoofinancials/cache.py
@@ -360,7 +360,6 @@ def initialise(self):
                 continue
         self.initialised = 0  # failure
 
-
     def lookup(self, strategy):
         if self.dummy:
             return None
diff --git a/yahoofinancials/data.py b/yahoofinancials/data.py
@@ -60,7 +60,7 @@ def get_data(self, session, url, request_headers=None, params=None, proxy=None,
         return response
 
 
-class YahooFinanceETL(object):
+class YahooFinanceData(object):
 
     def __init__(self, ticker, **kwargs):
         self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker]
@@ -72,6 +72,7 @@ def __init__(self, ticker, **kwargs):
         self.timeout = kwargs.get("timeout", 30)
         self.proxies = kwargs.get("proxies")
         self.session = kwargs.pop("session", None)
+        self.flat_format = kwargs.get("flat_format", False)
         self._cache = {}
 
     # Minimum interval between Yahoo Finance requests for this instance
@@ -167,9 +168,15 @@ def _construct_url(self, symbol, config, params, freq, request_type):
                 params.update({k: v['options'][request_type].get(freq)})
             elif k == "modules" and request_type in v['options']:
                 params.update({k: request_type})
+            elif k == "symbol":
+                params.update({k: symbol.lower()})
             elif k not in params:
+                if k == 'reportsCount' and v is None:
+                    continue
                 params.update({k: v['default']})
         for k, v in _default_query_params.items():  # general defaults
+            if k == 'reportsCount' and v is None:
+                continue
             if k not in params:
                 params.update({k: v})
         if params.get("type"):
@@ -183,6 +190,8 @@ def _construct_url(self, symbol, config, params, freq, request_type):
             for k, v in params.items():
                 if k != "modules":
                     url += "&" + k + "=" + str(v)
+        elif params.get("symbol"):
+            url += "?symbol=" + params.get("symbol")
         return url
 
     # Private method to execute a web scrape request and decrypt the return
@@ -269,6 +278,11 @@ def _get_historical_data(self, url, config, tech_type, statement_type):
         data = self._cache[url]
         if tech_type == '' and statement_type in ["income", "balance", "cash"]:
             data = self._format_raw_fundamental_data(data)
+        elif statement_type == 'analytic':
+            data = data.get("result")
+            if tech_type == "recommendations":
+                if isinstance(data, list) and len(data) > 0:
+                    data[0].get("recommendedSymbols")
         else:
             data = self._format_raw_module_data(data, tech_type)
         return data
@@ -519,7 +533,9 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi
             dict_ent = {}
             params = {}
             r_map = get_request_config(tech_type, REQUEST_MAP)
-            r_cat = get_request_category(tech_type, self.YAHOO_FINANCIAL_TYPES, statement_type)
+            r_cat = None
+            if statement_type != 'analytic':
+                r_cat = get_request_category(tech_type, self.YAHOO_FINANCIAL_TYPES, statement_type)
             YAHOO_URL = self._construct_url(
                 up_ticker.lower(),
                 r_map,
@@ -544,6 +560,17 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi
                 dict_ent = {up_ticker: re_data}
             return dict_ent
 
+    def _retry_create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj):
+        i = 0
+        while i < 250:
+            try:
+                out = self._create_dict_ent(up_ticker, statement_type, tech_type, report_name, hist_obj)
+                return out
+            except:
+                time.sleep(random.randint(2, 10))
+                i += 1
+                continue
+
     # Private method to return the stmt_id for the reformat_process
     def _get_stmt_id(self, statement_type, raw_data):
         stmt_id = ''
@@ -568,8 +595,22 @@ def _reformat_stmt_data_process(raw_data):
         else:
             return raw_data
 
+    # Private Method for the Flat Reformat Process
+    @staticmethod
+    def _reformat_stmt_data_process_flat(raw_data):
+        final_data = {}
+        if raw_data is not None:
+            for date_key, data_item in raw_data.items():
+                final_data.update({date_key: data_item})
+            return final_data
+        else:
+            return raw_data
+
     # Private Method to return subdict entry for the statement reformat process
     def _get_sub_dict_ent(self, ticker, raw_data):
+        if self.flat_format:
+            form_data_dict = self._reformat_stmt_data_process_flat(raw_data[ticker])
+            return {ticker: form_data_dict}
         form_data_list = self._reformat_stmt_data_process(raw_data[ticker])
         return {ticker: form_data_list}
 
@@ -581,17 +622,17 @@ def get_time_code(self, time_interval):
     # Public Method to get stock data
     def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}):
         data = {}
-        if statement_type == 'income' and tech_type == '' and report_name == '':    # temp, so this method doesn't return nulls
+        if statement_type == 'income' and tech_type == '' and report_name == '':  # temp, so this method doesn't return nulls
             statement_type = 'profile'
             tech_type = 'assetProfile'
             report_name = 'assetProfile'
         if isinstance(self.ticker, str):
-            dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj)
+            dict_ent = self._retry_create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj)
             data.update(dict_ent)
         else:
             if self.concurrent:
                 with Pool(self._get_worker_count()) as pool:
-                    dict_ents = pool.map(partial(self._create_dict_ent,
+                    dict_ents = pool.map(partial(self._retry_create_dict_ent,
                                                  statement_type=statement_type,
                                                  tech_type=tech_type,
                                                  report_name=report_name,
diff --git a/yahoofinancials/maps.py b/yahoofinancials/maps.py
@@ -2375,6 +2375,19 @@
             "padTimeSeries": {"required": False, "default": False},
         },
     },
+    "insights": {
+        "path": "https://query1.finance.yahoo.com/ws/insights/v2/finance/insights",
+        "response_field": "finance",
+        "request": {
+            "symbol": {"required": True, "default": None},
+            "reportsCount": {"required": False, "default": None},
+        },
+    },
+    "recommendations": {
+        "path": "https://query1.finance.yahoo.com/v6/finance/recommendationsbysymbol/{symbol}",
+        "response_field": "finance",
+        "request": {},
+    },
 }
 
 USER_AGENTS = [
diff --git a/yahoofinancials/sessions.py b/yahoofinancials/sessions.py
diff --git a/yahoofinancials/utils.py b/yahoofinancials/utils.py
diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py
