From 98ec2959f0811547f5257dd6698a231816e87097 Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Fri, 21 Apr 2023 13:10:53 -0500 Subject: [PATCH] Hotfix/ table fixes (#4848) * Update helper_funcs.py * init * Update test_display_cashflow_comparison.txt * Update test_display_cashflow_comparison.txt * fix copy paste fail * Update helper_funcs.py * Update backend.py --- frontend-components/tables/package-lock.json | 13 ++ frontend-components/tables/package.json | 1 + .../src/components/Table/ColumnHeader.tsx | 14 +- .../tables/src/components/Table/index.tsx | 67 ++++++--- frontend-components/tables/src/index.css | 20 +++ frontend-components/tables/src/utils/utils.ts | 142 ++++++++++++++---- .../alternative/covid/covid_model.py | 2 +- openbb_terminal/core/plots/backend.py | 6 +- openbb_terminal/core/plots/table.html | 98 ++++++------ .../econometrics/econometrics_controller.py | 2 +- openbb_terminal/helper_funcs.py | 8 + .../comparison_analysis/marketwatch_view.py | 5 + .../test_display_cashflow_comparison.txt | 45 +++--- 13 files changed, 293 insertions(+), 130 deletions(-) diff --git a/frontend-components/tables/package-lock.json b/frontend-components/tables/package-lock.json index bd904d64aaa1..48150335bd16 100644 --- a/frontend-components/tables/package-lock.json +++ b/frontend-components/tables/package-lock.json @@ -34,6 +34,7 @@ "@types/react": "^18.0.27", "@types/react-dom": "^18.0.10", "@types/react-table": "^7.7.14", + "@types/wicg-file-system-access": "^2020.9.6", "@vitejs/plugin-react": "^3.1.0", "autoprefixer": "^10.4.13", "clsx": "^1.2.1", @@ -1681,6 +1682,12 @@ "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", "devOptional": true }, + "node_modules/@types/wicg-file-system-access": { + "version": "2020.9.6", + "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.6.tgz", + "integrity": "sha512-6hogE75Hl2Ov/jgp8ZhDaGmIF/q3J07GtXf8nCJCwKTHq7971po5+DId7grft09zG7plBwpF6ZU0yx9Du4/e1A==", + "dev": true + }, "node_modules/@vitejs/plugin-react": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-3.1.0.tgz", @@ -6568,6 +6575,12 @@ "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", "devOptional": true }, + "@types/wicg-file-system-access": { + "version": "2020.9.6", + "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.6.tgz", + "integrity": "sha512-6hogE75Hl2Ov/jgp8ZhDaGmIF/q3J07GtXf8nCJCwKTHq7971po5+DId7grft09zG7plBwpF6ZU0yx9Du4/e1A==", + "dev": true + }, "@vitejs/plugin-react": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-3.1.0.tgz", diff --git a/frontend-components/tables/package.json b/frontend-components/tables/package.json index 531e986da4ff..5ee4b0a91f60 100644 --- a/frontend-components/tables/package.json +++ b/frontend-components/tables/package.json @@ -37,6 +37,7 @@ "@types/react": "^18.0.27", "@types/react-dom": "^18.0.10", "@types/react-table": "^7.7.14", + "@types/wicg-file-system-access": "^2020.9.6", "@vitejs/plugin-react": "^3.1.0", "autoprefixer": "^10.4.13", "clsx": "^1.2.1", diff --git a/frontend-components/tables/src/components/Table/ColumnHeader.tsx b/frontend-components/tables/src/components/Table/ColumnHeader.tsx index 51de21751d48..511dcec7e9a4 100644 --- a/frontend-components/tables/src/components/Table/ColumnHeader.tsx +++ b/frontend-components/tables/src/components/Table/ColumnHeader.tsx @@ -1,8 +1,10 @@ import { flexRender } from "@tanstack/react-table"; import clsx from "clsx"; import { FC } from "react"; +import { includesDateNames } from "../../utils/utils"; import { useDrag, useDrop } from "react-dnd"; import * as ContextMenuPrimitive from "@radix-ui/react-context-menu"; + function Filter({ column, table, @@ -29,9 +31,15 @@ function Filter({ const columnFilterValue = column.getFilterValue(); - const isProbablyDate = - column.id.toLowerCase().includes("date") || - (column.id.toLowerCase() === "index" && !valuesContainStringWithSpaces); + const isProbablyDate = values.every((value) => { + if (typeof value !== "string") return false; + const only_numbers = value.replace(/[^0-9]/g, ""); + return ( + only_numbers.length >= 4 && + (includesDateNames(column.id) || + (column.id.toLowerCase() === "index" && !valuesContainStringWithSpaces)) + ); + }); if (isProbablyDate) { function getTime(value) { diff --git a/frontend-components/tables/src/components/Table/index.tsx b/frontend-components/tables/src/components/Table/index.tsx index 57e0998ac398..b12b03eabefd 100644 --- a/frontend-components/tables/src/components/Table/index.tsx +++ b/frontend-components/tables/src/components/Table/index.tsx @@ -11,7 +11,12 @@ import clsx from "clsx"; import { useMemo, useRef, useState } from "react"; import { useVirtual } from "react-virtual"; import Select from "../Select"; -import { formatNumberMagnitude, fuzzyFilter, isEqual } from "../../utils/utils"; +import { + formatNumberMagnitude, + fuzzyFilter, + isEqual, + includesDateNames, +} from "../../utils/utils"; import DraggableColumnHeader from "./ColumnHeader"; import Pagination, { validatePageSize } from "./Pagination"; import Export from "./Export"; @@ -39,20 +44,24 @@ function getCellWidth(row, column) { const indexValue = indexLabel ? row[indexLabel] : null; const value = row[column]; const valueType = typeof value; + const only_numbers = value?.toString().replace(/[^0-9]/g, ""); + const probablyDate = - column.toLowerCase().includes("date") || - column.toLowerCase() === "index" || - (indexValue && - typeof indexValue == "string" && - (indexValue.toLowerCase().includes("date") || - indexValue.toLowerCase().includes("day") || - indexValue.toLowerCase().includes("time") || - indexValue.toLowerCase().includes("timestamp") || - indexValue.toLowerCase().includes("year") || - indexValue.toLowerCase().includes("month") || - indexValue.toLowerCase().includes("week") || - indexValue.toLowerCase().includes("hour") || - indexValue.toLowerCase().includes("minute"))); + only_numbers.length >= 4 && + (includesDateNames(column) || + column.toLowerCase() === "index" || + (indexValue && + indexValue && + typeof indexValue == "string" && + (indexValue.toLowerCase().includes("date") || + indexValue.toLowerCase().includes("day") || + indexValue.toLowerCase().includes("time") || + indexValue.toLowerCase().includes("timestamp") || + indexValue.toLowerCase().includes("year") || + indexValue.toLowerCase().includes("month") || + indexValue.toLowerCase().includes("week") || + indexValue.toLowerCase().includes("hour") || + indexValue.toLowerCase().includes("minute")))); const probablyLink = valueType === "string" && value.startsWith("http"); if (probablyLink) { @@ -165,16 +174,17 @@ export default function Table({ const indexValue = indexLabel ? row.original[indexLabel] : null; const value = row.original[column]; const valueType = typeof value; + const only_numbers = value?.toString().replace(/[^0-9]/g, ""); const probablyDate = - column.toLowerCase().includes("date") || - column.toLowerCase().includes("timestamp") || - column.toLowerCase() === "index" || - (indexValue && - typeof indexValue == "string" && - (indexValue.toLowerCase().includes("date") || - indexValue.toLowerCase().includes("time") || - indexValue.toLowerCase().includes("timestamp") || - indexValue.toLowerCase().includes("year"))); + only_numbers.length >= 4 && + (includesDateNames(column) || + column.toLowerCase() === "index" || + (indexValue && + typeof indexValue == "string" && + (indexValue.toLowerCase().includes("date") || + indexValue.toLowerCase().includes("time") || + indexValue.toLowerCase().includes("timestamp") || + indexValue.toLowerCase().includes("year")))); const probablyLink = valueType === "string" && value.startsWith("http"); @@ -193,7 +203,16 @@ export default function Table({ } if (probablyDate) { if (typeof value === "string") { - return

{value}

; + const date = value.split("T")[0]; + const time = value.split("T")[1]?.split(".")[0]; + if (time === "00:00:00") { + return

{date}

; + } + return ( +

+ {date} {time} +

+ ); } if (typeof value === "number") { diff --git a/frontend-components/tables/src/index.css b/frontend-components/tables/src/index.css index ea20d8ae7ce4..58d8a6b6481b 100644 --- a/frontend-components/tables/src/index.css +++ b/frontend-components/tables/src/index.css @@ -95,3 +95,23 @@ tr { opacity: 1; } } + +/* custom scrollbar */ +::-webkit-scrollbar { + width: 20px; +} + +::-webkit-scrollbar-track { + background-color: transparent; +} + +::-webkit-scrollbar-thumb { + background-color: #d6dee1; + border-radius: 20px; + border: 6px solid transparent; + background-clip: content-box; +} + +::-webkit-scrollbar-thumb:hover { + background-color: #a8bbbf; +} diff --git a/frontend-components/tables/src/utils/utils.ts b/frontend-components/tables/src/utils/utils.ts index 3d2b6afd7ea5..db5f6b4ac08c 100644 --- a/frontend-components/tables/src/utils/utils.ts +++ b/frontend-components/tables/src/utils/utils.ts @@ -3,25 +3,28 @@ import domtoimage from "dom-to-image"; import { utils, writeFile } from "xlsx"; export function formatNumberMagnitude(number: number) { - if (number < 10) { - return number.toFixed(4); - } else if (number < 100) { - return number.toFixed(3); - } else if (number < 1000) { - return number.toFixed(2); + if (number % 1 !== 0) { + const decimalPlaces = Math.max(2, number.toString().split(".")[1].length); + const toFixed = Math.min(4, decimalPlaces); + if (number < 1000) { + return number.toFixed(toFixed) || 0; + } } - const abs = Math.abs(number); - if (abs >= 1000000000000) { - return `${(number / 1000000000).toFixed(2)}T`; - } else if (abs >= 1000000000) { - return `${(number / 1000000000).toFixed(2)}B`; - } else if (abs >= 1000000) { - return `${(number / 1000000).toFixed(2)}M`; - } else if (abs >= 1000) { - return `${(number / 1000).toFixed(2)}K`; - } else { - return number.toFixed(2); + + if (number > 1000) { + const magnitude = Math.min(4, Math.floor(Math.log10(Math.abs(number)) / 3)); + const suffix = ["", "K", "M", "B", "T"][magnitude]; + const formatted = (number / 10 ** (magnitude * 3)).toFixed(2); + return `${formatted} ${suffix}`; } + + return number; +} + +export function includesDateNames(column: string) { + return ["date", "day", "time", "timestamp", "year"].some((dateName) => + column.toLowerCase().includes(dateName) + ); } export function isEqual(a: any, b: any) { @@ -46,7 +49,82 @@ export const fuzzyFilter = ( return itemRank; }; -export const saveToFile = (blob: Blob, fileName: string) => { +const writeFileHandler = async ({ + fileHandle, + blob, +}: { + fileHandle?: FileSystemFileHandle | null; + blob: Blob; +}) => { + if (!fileHandle) { + throw new Error("Cannot access filesystem"); + } + const writer = await fileHandle.createWritable(); + await writer.write(blob); + await writer.close(); +}; + +const IMAGE_TYPE: FilePickerAcceptType[] = [ + { + description: "PNG Image", + accept: { + "image/png": [".png"], + }, + }, + { + description: "JPEG Image", + accept: { + "image/jpeg": [".jpeg"], + }, + }, + { + description: "SVG Image", + accept: { + "image/svg+xml": [".svg"], + }, + }, +]; + +const getNewFileHandle = ({ + filename, + is_image, +}: { + filename: string; + is_image?: boolean; +}): Promise => { + if ("showSaveFilePicker"! in window) { + return new Promise((resolve) => { + resolve(null); + }); + } + + const opts: SaveFilePickerOptions = { + suggestedName: filename, + types: is_image + ? IMAGE_TYPE + : [ + { + description: "CSV File", + accept: { + "image/csv": [".csv"], + }, + }, + ], + excludeAcceptAllOption: true, + }; + + return showSaveFilePicker(opts); +}; + +export const saveToFile = ( + blob: Blob, + fileName: string, + fileHandle?: FileSystemFileHandle | null +) => { + if ("showSaveFilePicker" in window) { + return writeFileHandler({ fileHandle, blob }); + } + const url = URL.createObjectURL(blob); const link = document.createElement("a"); link.setAttribute("href", url); @@ -63,21 +141,33 @@ export const downloadData = (type: "csv" | "xlsx", columns: any, data: any) => { headers.map((column: any) => row[column]) ); const csvData = [headers, ...rows]; + if (type === "csv") { const csvContent = csvData.map((e) => e.join(",")).join("\n"); const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" }); - saveToFile(blob, `${window.title}.csv`); - } else { - const wb = utils.book_new(); - const ws = utils.aoa_to_sheet(csvData); - utils.book_append_sheet(wb, ws, "Sheet1"); - writeFile(wb, `${window.title}.xlsx`); + const filename = `${window.title}.csv`; + return getNewFileHandle({ + filename: filename, + }).then((fileHandle) => { + saveToFile(blob, filename, fileHandle); + }); } + + const wb = utils.book_new(); + const ws = utils.aoa_to_sheet(csvData); + utils.book_append_sheet(wb, ws, "Sheet1"); + writeFile(wb, `${window.title}.xlsx`); }; export const downloadImage = (id: string) => { const table = document.getElementById(id); - domtoimage.toBlob(table).then(function (blob) { - saveToFile(blob, `${window.title}.png`); + const filename = `${window.title}.png`; + getNewFileHandle({ + filename: filename, + is_image: true, + }).then((fileHandle) => { + domtoimage.toBlob(table).then(function (blob: Blob) { + saveToFile(blob, filename, fileHandle); + }); }); }; diff --git a/openbb_terminal/alternative/covid/covid_model.py b/openbb_terminal/alternative/covid/covid_model.py index 51da491487ff..7d5a66713376 100644 --- a/openbb_terminal/alternative/covid/covid_model.py +++ b/openbb_terminal/alternative/covid/covid_model.py @@ -110,7 +110,7 @@ def get_covid_ov( return pd.DataFrame() data = pd.concat([cases, deaths], axis=1) data.columns = ["Cases", "Deaths"] - data.index = [x.strftime("%Y-%m-%d") for x in data.index] + data.index = pd.to_datetime(data.index, format="%Y-%m-%d") return data diff --git a/openbb_terminal/core/plots/backend.py b/openbb_terminal/core/plots/backend.py index e7d9e585f2ed..3a4b94567072 100644 --- a/openbb_terminal/core/plots/backend.py +++ b/openbb_terminal/core/plots/backend.py @@ -262,13 +262,13 @@ def send_table( # pylint: disable=C0415 from openbb_terminal.helper_funcs import command_location - json_data = json.loads(df_table.to_json(orient="split")) + json_data = json.loads(df_table.to_json(orient="split", date_format="iso")) json_data.update( dict( title=title, source=source or "", theme=theme or "dark", - command_location=command_location or "", + command_location=command_location, ) ) @@ -279,7 +279,7 @@ def send_table( "json_data": json.dumps(json_data), "width": width, "height": self.HEIGHT - 100, - **self.get_kwargs(title), + **self.get_kwargs(command_location), } ) ) diff --git a/openbb_terminal/core/plots/table.html b/openbb_terminal/core/plots/table.html index 9083184614a1..6877d9566179 100644 --- a/openbb_terminal/core/plots/table.html +++ b/openbb_terminal/core/plots/table.html @@ -32,47 +32,47 @@ } diff --git a/openbb_terminal/econometrics/econometrics_controller.py b/openbb_terminal/econometrics/econometrics_controller.py index cf1233a1cf12..a88fa67be37a 100644 --- a/openbb_terminal/econometrics/econometrics_controller.py +++ b/openbb_terminal/econometrics/econometrics_controller.py @@ -621,7 +621,7 @@ def call_show(self, other_args: List[str]): df, headers=list(df.columns), show_index=True, - title=f"Dataset {name} | Showing {ns_parser.limit} of {len(df)} rows", + title=f"Dataset {name}", export=bool(ns_parser.export), limit=ns_parser.limit, ) diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 5cbb2549a015..2a7b58e5ec4f 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -301,6 +301,14 @@ def print_rich_table( current_user.preferences.USE_INTERACTIVE_DF and plots_backend().isatty ) + show_index = not isinstance(df.index, pd.RangeIndex) and show_index + + for col in df.columns: + try: + df[col] = pd.to_numeric(df[col]) + except ValueError: + pass + def _get_headers(_headers: Union[List[str], pd.Index]) -> List[str]: """Check if headers are valid and return them.""" output = _headers diff --git a/openbb_terminal/stocks/comparison_analysis/marketwatch_view.py b/openbb_terminal/stocks/comparison_analysis/marketwatch_view.py index bc817d867c5b..2c83f4ceccb1 100644 --- a/openbb_terminal/stocks/comparison_analysis/marketwatch_view.py +++ b/openbb_terminal/stocks/comparison_analysis/marketwatch_view.py @@ -185,6 +185,11 @@ def display_cashflow_comparison( if not quarter: df_financials_compared.index.name = timeframe + if any(isinstance(col, tuple) for col in df_financials_compared.columns): + df_financials_compared.columns = [ + " ".join(col) for col in df_financials_compared.columns + ] + print_rich_table( df_financials_compared, headers=list(df_financials_compared.columns), diff --git a/tests/openbb_terminal/stocks/comparison_analysis/txt/test_marketwatch_view/test_display_cashflow_comparison.txt b/tests/openbb_terminal/stocks/comparison_analysis/txt/test_marketwatch_view/test_display_cashflow_comparison.txt index 6471ee4e158c..a4c139b0596b 100644 --- a/tests/openbb_terminal/stocks/comparison_analysis/txt/test_marketwatch_view/test_display_cashflow_comparison.txt +++ b/tests/openbb_terminal/stocks/comparison_analysis/txt/test_marketwatch_view/test_display_cashflow_comparison.txt @@ -1,23 +1,22 @@ -Other available quarterly timeframes are: 30-Sep-2021, 31-Dec-2021, 31-Mar-2022, 30-Jun-2022, 30-Sep-2022 - - 31-Dec-2021 - TSLA GM -Item -Net Income before Extraordinaries 2.34B 1.77B -Net Income Growth 41.23% -25.98% -Depreciation, Depletion & Amortization 848M 2.94B -Depreciation and Depletion 530M - -Amortization of Intangible Assets 318M - -Deferred Taxes & Investment Tax Credit - 251M -Deferred Taxes - 251M -Investment Tax Credit - - -Other Funds 539M (783M) -Funds from Operations 3.73B 4.17B -Extraordinaries - - -Changes in Working Capital 855M 2.64B -Receivables 18M - -Accounts Payable 1.81B - -Other Assets/Liabilities (443M) 4.59B -Net Operating Cash Flow 4.59B 6.81B -Net Operating Cash Flow Growth 45.69% 13,995.92% -Net Operating Cash Flow / Sales 25.88% 20.27% +Other available quarterly timeframes are: 30-Sep-2021, 31-Dec-2021, 31-Mar-2022, 30-Jun-2022, 30-Sep-2022 + + 31-Dec-2021 TSLA 31-Dec-2021 GM +Item +Net Income before Extraordinaries 2.34B 1.77B +Net Income Growth 41.23% -25.98% +Depreciation, Depletion & Amortization 848M 2.94B +Depreciation and Depletion 530M - +Amortization of Intangible Assets 318M - +Deferred Taxes & Investment Tax Credit - 251M +Deferred Taxes - 251M +Investment Tax Credit - - +Other Funds 539M (783M) +Funds from Operations 3.73B 4.17B +Extraordinaries - - +Changes in Working Capital 855M 2.64B +Receivables 18M - +Accounts Payable 1.81B - +Other Assets/Liabilities (443M) 4.59B +Net Operating Cash Flow 4.59B 6.81B +Net Operating Cash Flow Growth 45.69% 13,995.92% +Net Operating Cash Flow / Sales 25.88% 20.27%