aihub_json_preprocessing_document_summary_text.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AIHub Json Preprocessing"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Development Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install kss==3.7.3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS Argument Error: Restart Jupyter Kernel Runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install python-mecab-ko"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS 3.7.3 matches python-mecab-ko"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install ray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import kss\n",
    "import ray\n",
    "import json\n",
    "import time\n",
    "from time import sleep\n",
    "from tqdm import tqdm\n",
    "from mecab import MeCab\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c:\\\\Users\\\\MinSeok\\\\Documents\\\\AIHUB'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AIHUB 문서요약 텍스트"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Source](https://aihub.or.kr/aihubdata/data/view.do?currMenu=115&topMenu=100&aihubDataSe=realm&dataSetSn=97)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert JSON File to TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_preprocessing import make_train_valid_json_txt_file_path_list\n",
    "from data_preprocessing import count_number_of_txt_file_with_batch_list\n",
    "from data_preprocessing import write_jsontext_to_txt_file_with_batch_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_path_list = ['AIHUB_문서요약 텍스트/Training/'+ '/**/*.json', \n",
    "                  'AIHUB_문서요약 텍스트/Validation/'+ '/**/*.json']\n",
    "txt_path_list = [\"exploration/document_summary_text_pro/AIHUB_document_summary_text_train_\", \n",
    "                 \"exploration/document_summary_text_pro/AIHUB_document_summary_text_valid_\"]\n",
    "corpus_name = \"document_summary_text\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of file: 6\n"
     ]
    }
   ],
   "source": [
    "train_json_file_list, valid_json_file_list, train_txt_file_path_list, valid_txt_file_path_list = \\\n",
    "    make_train_valid_json_txt_file_path_list(json_path_list, txt_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 2474\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = 'document_summary_text'\n",
    "the_number_of_train_txt_file, the_number_of_train_txt_file_list = count_number_of_txt_file_with_batch_list(train_json_file_list, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>File</th>\n",
       "      <th>Length of Source List</th>\n",
       "      <th>The Number of txt File</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AIHUB_문서요약 텍스트/Training\\법률_train_original\\trai...</td>\n",
       "      <td>42178</td>\n",
       "      <td>43</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AIHUB_문서요약 텍스트/Training\\사설_train_original\\trai...</td>\n",
       "      <td>242550</td>\n",
       "      <td>243</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AIHUB_문서요약 텍스트/Training\\신문기사_train_original\\tr...</td>\n",
       "      <td>2187380</td>\n",
       "      <td>2188</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               File  \\\n",
       "0           0  AIHUB_문서요약 텍스트/Training\\법률_train_original\\trai...   \n",
       "1           1  AIHUB_문서요약 텍스트/Training\\사설_train_original\\trai...   \n",
       "2           2  AIHUB_문서요약 텍스트/Training\\신문기사_train_original\\tr...   \n",
       "\n",
       "   Length of Source List  The Number of txt File  Description  \n",
       "0                  42178                      43          NaN  \n",
       "1                 242550                     243          NaN  \n",
       "2                2187380                    2188          NaN  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_file_by_batch_train_df = pd.read_excel('source_file_by_batch/document_summary_text_train.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = 'document_summary_text'\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_list, train_txt_file_path_list, batch_size, the_number_of_train_txt_file_list, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 271\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = 'document_summary_text'\n",
    "the_number_of_valid_txt_file, the_number_of_valid_txt_file_list = count_number_of_txt_file_with_batch_list(valid_json_file_list, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>File</th>\n",
       "      <th>Length of Source List</th>\n",
       "      <th>The Number of txt File</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AIHUB_문서요약 텍스트/Validation\\법률_valid_original\\va...</td>\n",
       "      <td>4780</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AIHUB_문서요약 텍스트/Validation\\사설_valid_original\\va...</td>\n",
       "      <td>25202</td>\n",
       "      <td>26</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AIHUB_문서요약 텍스트/Validation\\신문기사_valid_original\\...</td>\n",
       "      <td>239892</td>\n",
       "      <td>240</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               File  \\\n",
       "0           0  AIHUB_문서요약 텍스트/Validation\\법률_valid_original\\va...   \n",
       "1           1  AIHUB_문서요약 텍스트/Validation\\사설_valid_original\\va...   \n",
       "2           2  AIHUB_문서요약 텍스트/Validation\\신문기사_valid_original\\...   \n",
       "\n",
       "   Length of Source List  The Number of txt File  Description  \n",
       "0                   4780                       5          NaN  \n",
       "1                  25202                      26          NaN  \n",
       "2                 239892                     240          NaN  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_file_by_batch_valid_df = pd.read_excel('source_file_by_batch/document_summary_text_valid.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Size]\n",
      "The number of preprocessing corpus: 271\n",
      "\n",
      "[Order]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 271/271 [00:26<00:00, 10.24it/s]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = 'document_summary_text'\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_list, valid_txt_file_path_list, batch_size, the_number_of_valid_txt_file_list, corpus_name)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_segmentation import preprocessing_text\n",
    "from data_preprocessing import make_pro_post_txt_file_path_list\n",
    "from data_preprocessing import merge_and_deduplicate_corpus_txt\n",
    "from reading_data import reading_txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_corpus_path = \"AIHUB_corpus/exploration/document_summary_text_pro/AIHUB_document_summary_text_\" + \"*.txt\"\n",
    "pro_post_xlsx_path = \"pro_post_corpus_path/document_summary_text.xlsx\"\n",
    "pro_total_corpus_path_list, post_total_corpus_path_list = make_pro_post_txt_file_path_list(pro_corpus_path, pro_post_xlsx_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2745"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pro_total_corpus_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 가. 부가가치세법 제22조 제3항 단서에 제1호와 제2호가 동시에 해당한다는 뜻은 제18조의 예정신고와 예정납부끼리, 제19조의 확정신고와 확정납부끼리 동시에 해당하는 경우를 말하는 것이지 제18조의 예정신고나 그 납부와 제19조의 확정신고나 그 납부가 동시에 해당하는 경우를 가리키는 것이 아니라고 해석되므로 부가가치세의 예정신고 또는 그 납부를 아니하고 또한 확정신고 또는 그 납부를 아니한 경우 통털어서 한번만 가산세를 부과하는 것이 아니라 각각 독립하여 가산세부과대상이 된다.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"source\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "부가가치세법 제22조 제3항 단서에 제1호와 제2호가 동시에 해당한다는 뜻은 제18조의 예정신고와 예정납부끼리, 제19조의 확정신고와 확정납부끼리 동시에 해당하는 경우를 말하는 것이지 제18조의 예정신고나 그 납부와 제19조의 확정신고나 그 납부가 동시에 해당하는 경우를 가리키는 것이 아니라고 해석되므로 부가가치세의 예정신고 또는 그 납부를 아니하고 또한 확정신고 또는 그 납부를 아니한 경우 통털어서 한번만 가산세를 부과하는 것이 아니라 각각 독립하여 가산세부과대상이 된다.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"preprocessing\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-31 17:08:20,288\tINFO worker.py:1625 -- Started a local Ray instance.\n"
     ]
    }
   ],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/document_summary_text.xlsx' \\\n",
    "    --start_index 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessing_corpus_path = \"AIHUB_corpus/exploration/document_summary_text_post/AIHUB_document_summary_text_\" +\"*.txt\"\n",
    "merge_corpus_path = 'AIHUB_corpus/duplicate/AIHUB_document_summary_text.txt'\n",
    "deduplicate_corpus_path = 'AIHUB_corpus/AIHUB_document_summary_text.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_and_deduplicate_corpus_txt(preprocessing_corpus_path, merge_corpus_path, \n",
    "                                  deduplicate_corpus_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "corpus_05",
   "language": "python",
   "name": "corpus_05"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}