Initial commit

Author: Olegas

.gitignore

@@ -0,0 +1,4 @@
+.userData
+.idea
+node_modules
+out-inc-state-*.pdf

.prettierrc

@@ -0,0 +1,7 @@
+{
+  "printWidth": 90,
+  "tabWidth": 4,
+  "singleQuote": true,
+  "trailingComma": "none",
+  "bracketSpacing": false
+}

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "3-ndfl-fill-dividents",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "start": "node ./src/index.ts"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "currency-codes": "^2.1.0",
+    "i18n-iso-countries": "^7.2.0",
+    "node-fetch": "^2.0.0",
+    "pdf2json": "^2.0.0",
+    "puppeteer": "^13.1.2",
+    "ts-node": "^10.4.0"
+  },
+  "devDependencies": {
+    "@types/node": "^17.0.13",
+    "@types/node-fetch": "^3.0.3",
+    "prettier": "^2.5.1",
+    "typescript": "^4.5.5"
+  }
+}

src/app.ts

@@ -0,0 +1,94 @@
+import {TinkoffForeignTaxReportParser} from './reportParser';
+import {EntryValidatory} from './validator';
+import {ReportEnricher} from './reportEnricher';
+import {NalogRuReportMaker} from './reportMaker';
+import {NalogRuPageController} from './nalogRuPageController';
+import {EnrichedTaxEntry, TaxEntry} from './entities';
+import path from 'path';
+import puppeteer from 'puppeteer';
+
+export class App {
+    constructor(private _fileName: string) {}
+
+    async run() {
+        const items = await this._parseReport();
+        const enrichedItems = await this._enrichItems(items);
+        const browserController = new NalogRuPageController();
+        try {
+            const page = await this._preparePage(browserController);
+            await this._makeReport(page, enrichedItems);
+        } catch (e: any) {
+            console.log('[❌] Something went wrong...');
+            console.log(`[❌] Error details: ${e.message}`);
+            console.log('[❌] Screenshot is saved');
+            console.log('[❌] Press Enter to close browser');
+            await this._readEnter();
+            await browserController.screenshot(path.join(process.cwd(), 'error.png'));
+        } finally {
+            await browserController.finalize();
+        }
+    }
+
+    private async _parseReport() {
+        console.log('[🚀] Parsing tax report...');
+        const report = new TinkoffForeignTaxReportParser(this._fileName);
+        const items = await report.getTaxEntries();
+        const itemsValid = items.every((i) => EntryValidatory.isValid(i));
+        if (!itemsValid) {
+            throw new Error('Invalid items found');
+        }
+        console.log(`[✅] Done! ${items.length} parsed`);
+        return items;
+    }
+
+    private async _enrichItems(items: TaxEntry[]) {
+        console.log(
+            `[🚀] Enriching items (loading company names). This may take a while...`
+        );
+        const enricher = new ReportEnricher();
+        const enrichedItems = await enricher.enrichItems(items);
+        console.log(`[✅] Done!`);
+        return enrichedItems;
+    }
+
+    private async _preparePage(browserController: NalogRuPageController) {
+        console.log('[❗️] After pressing Enter, browser will be started...');
+        console.log(
+            '[❗️] Login with any suitable method, when return back and press Enter once again'
+        );
+        await this._readEnter();
+        console.log('[🚀] Loading browser to make report...');
+        const page = await browserController.getNalogRuPage();
+        console.log(
+            '[🖐] Please login to your Nalog.RU account, when go back here and press Enter'
+        );
+        await this._readEnter();
+
+        return page;
+    }
+
+    private async _makeReport(page: puppeteer.Page, items: EnrichedTaxEntry[]) {
+        console.log('[🚀] Making report. Please wait, this will take a while...');
+        const reportMaker = new NalogRuReportMaker(page);
+        await reportMaker.makeReport(items);
+        console.log(`[✅] Done! Draft URL: ${page.url()}`);
+    }
+
+    private async _readEnter() {
+        const stdin = process.stdin;
+        return new Promise<void>((res) => {
+            const handler = (key: Buffer) => {
+                if (key.toString() === '\r') {
+                    stdin.setRawMode(false);
+                    stdin.pause();
+                    stdin.off('data', handler);
+                    res();
+                }
+            };
+            stdin.setRawMode(true);
+            stdin.resume();
+            stdin.setEncoding('utf8');
+            stdin.on('data', handler);
+        });
+    }
+}

src/entities/index.ts

@@ -0,0 +1,19 @@
+export interface TaxEntry {
+    dateFix: Date;
+    datePay: Date;
+    isin: string;
+    country: string;
+    count: number;
+    cost: number;
+    comission: number;
+    sumBeforeTax: number;
+    tax: number;
+    sumAfterTax: number;
+    currency: string;
+}
+
+export interface EnrichedTaxEntry extends TaxEntry {
+    countryCode: number;
+    currencyCode: number;
+    name: string;
+}

src/index.js

@@ -0,0 +1,6 @@
+require('ts-node/register');
+const {App} = require('./app');
+const path = require('path');
+
+const app = new App(path.join(process.cwd(), process.argv[2]));
+app.run();

src/nalogRuPageController/index.ts

@@ -0,0 +1,47 @@
+import puppeteer from 'puppeteer';
+import path from 'path';
+import fs from 'fs';
+
+const LK_URL = 'https://lkfl2.nalog.ru/lkfl/login';
+
+export class NalogRuPageController {
+    private _browser: puppeteer.Browser | undefined;
+    private _page: puppeteer.Page | undefined;
+
+    async getNalogRuPage(): Promise<puppeteer.Page> {
+        const browser = await this._createBrowser();
+        const page = await browser.newPage();
+        await page.setViewport({
+            width: 1024,
+            height: 768
+        });
+        this._page = page;
+        await Promise.all([this._page.waitForNavigation(), this._page.goto(LK_URL)]);
+        return page;
+    }
+
+    async screenshot(path: string) {
+        await this._page?.screenshot({
+            type: 'png',
+            path
+        });
+    }
+
+    async finalize() {
+        await this._page?.close();
+        await this._browser?.close();
+    }
+
+    private async _createBrowser(): Promise<puppeteer.Browser> {
+        const userDataDir = path.join(process.cwd(), '.userData');
+        if (!fs.existsSync(userDataDir)) {
+            fs.mkdirSync(userDataDir);
+        }
+        this._browser = await puppeteer.launch({
+            headless: false,
+            slowMo: 10,
+            userDataDir
+        });
+        return this._browser;
+    }
+}

src/reportEnricher/index.ts

@@ -0,0 +1,99 @@
+import {EnrichedTaxEntry, TaxEntry} from '../entities';
+import countries from 'i18n-iso-countries';
+// @ts-ignore
+import ru from 'i18n-iso-countries/langs/ru';
+import currencyCodes from 'currency-codes';
+// @ts-ignore
+import fetch from 'node-fetch';
+
+countries.registerLocale(ru);
+
+function delay(ms: number) {
+    return new Promise((res) => setTimeout(res, ms));
+}
+
+export class ReportEnricher {
+    private _cache: Record<string, string> = {};
+
+    async enrichItems(items: TaxEntry[]): Promise<EnrichedTaxEntry[]> {
+        const result: EnrichedTaxEntry[] = [];
+        await this._preloadIsinCache(items);
+        for (const item of items) {
+            result.push(await this._enrichItem(item));
+        }
+        return result;
+    }
+
+    private async _preloadIsinCache(items: TaxEntry[]) {
+        const uniqueIsin = [...new Set(items.map((i) => i.isin))];
+        const batches = uniqueIsin.reduce(
+            (batches: Array<string[]>, isin) => {
+                const lastBatch = batches[batches.length - 1];
+                if (lastBatch.length < 10) {
+                    lastBatch.push(isin);
+                } else {
+                    batches.push([isin]);
+                }
+                return batches;
+            },
+            [[]]
+        );
+        for (const batch of batches) {
+            this._cache = {
+                ...this._cache,
+                ...(await this._preloadBatch(batch))
+            };
+        }
+    }
+
+    private async _preloadBatch(batch: string[]): Promise<Record<string, string>> {
+        let response;
+        const batchResult: Record<string, string> = {};
+        while (true) {
+            response = await fetch('https://api.openfigi.com/v3/mapping', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify(
+                    batch.map((isin) => ({idType: 'ID_ISIN', idValue: isin}))
+                )
+            });
+            if (response.status === 429) {
+                console.warn('[⏱] Rate-limited, sleeping...');
+                await delay(10000);
+                continue;
+            }
+            break;
+        }
+        const result = await response.json();
+        for (let i = 0; i < batch.length; i++) {
+            const {
+                data: [{name}]
+            } = result[i];
+            batchResult[batch[i]] = name;
+        }
+        return batchResult;
+    }
+
+    private _getName(item: TaxEntry): string {
+        return this._cache[item.isin];
+    }
+
+    private async _enrichItem(item: TaxEntry): Promise<EnrichedTaxEntry> {
+        const alpha2 = countries.getAlpha2Code(item.country, 'ru');
+        if (!alpha2) {
+            throw new Error(`Failed to detect alpha-2 code for country ${item.country}`);
+        }
+        const currencyCode = currencyCodes.code(item.currency)?.number;
+        if (!currencyCode) {
+            throw new Error(`Failed to detect code for currency ${item.currency}`);
+        }
+        return {
+            ...item,
+            countryCode: +countries.alpha2ToNumeric(alpha2),
+            currencyCode: +currencyCode,
+            name: this._getName(item)
+        };
+    }
+}