', s)
+ return s
+
+
+def str2cell(s):
+ value, refs = extract_references(s)
+ value = remove_text_styles(value)
+ value = unidecode(value)
+ return Cell(value=value, raw_value=s, refs=refs)
+
+def read_str_csv(filename):
+ try:
+ df = pd.read_csv(filename, header=None, dtype=str, keep_default_na=False)
+ except pd.errors.EmptyDataError:
+ df = pd.DataFrame()
+ return df
+
+
+class CellDataFrame(pd.DataFrame):
+ """We subclass pandas DataFrame in order to make deepcopy recursively copy cells"""
+ def __deepcopy__(self, memodict={}):
+ return CellDataFrame(self.applymap(lambda cell: deepcopy(cell, memodict)))
+
+
+class Table:
+ def __init__(self, name, df, layout, caption=None, figure_id=None, annotations=None, migrate=False, old_name=None, guessed_tags=None):
+ self.name = name
+ self.caption = caption
+ self.figure_id = figure_id
+ self.df = CellDataFrame(df.applymap(str2cell))
+
+ if migrate:
+ self.old_name = old_name
+
+ if layout is not None:
+ self.set_layout(layout)
+
+ self._set_annotations(annotations, migrate=migrate, old_name=old_name, guessed_tags=guessed_tags)
+
+ def _set_annotations(self, annotations, migrate=False, old_name=None, guessed_tags=None):
+ if annotations is not None:
+ self.gold_tags = annotations.gold_tags.strip()
+ self.dataset_text = annotations.dataset_text.strip()
+ self.notes = annotations.notes.strip()
+
+ sota_records = json.loads(annotations.cells_sota_records)
+
+ if guessed_tags is not None:
+ tags = guessed_tags.values
+ else:
+ tags = annotations.matrix_gold_tags
+ gt_rows = len(tags)
+ if gt_rows == 0 and len(self.df) > 0:
+ print(f"Gold tags size mismatch: 0 vs {len(self.df)} in old name {old_name}")
+ if migrate:
+ self.old_name = None
+ elif gt_rows > 0:
+ gt_cols = len(tags[0])
+ if self.df.shape != (0,0) and self.df.shape == (gt_rows, gt_cols):
+ self.set_tags(tags)
+ else:
+ print(f"Gold tags size mismatch: {gt_rows},{gt_cols} vs {self.df.shape}")
+ # print(f"Gold tags size mismatch: {gt_rows},{gt_cols} vs {self.df.shape}")
+ # print(annotations.matrix_gold_tags)
+ # print(self.df.applymap(lambda c:c.value))
+ if migrate:
+ self.old_name = None
+ else:
+ self.gold_tags = ''
+ self.dataset_text = ''
+ self.notes = ''
+ sota_records = {}
+
+ sota_records = pd.DataFrame(sota_records.values(), index=sota_records.keys(),
+ columns=['task', 'dataset', 'metric', 'format', 'model', 'value'])
+ sota_records.index = self.name + "/" + sota_records.index
+ sota_records.index.rename("cell_ext_id", inplace=True)
+ sota_records.rename(columns={"value": "raw_value"}, inplace=True)
+
+ self.sota_records = sota_records.replace("", np.nan).dropna(subset=["model", "metric", "task", "dataset"])
+
+
+ def set_layout(self, layout):
+ for r, row in layout.iterrows():
+ for c, cell in enumerate(row):
+ self.df.iloc[r, c].layout = cell
+
+ def set_tags(self, tags):
+ for r, row in enumerate(tags):
+ for c, cell in enumerate(row):
+ # todo: change gold_tags to tags to avoid confusion
+ self.df.iloc[r,c].gold_tags = cell.strip()
+
+ @property
+ def shape(self):
+ return self.df.shape
+
+ @property
+ def matrix(self):
+ return self.df.applymap(lambda x: x.value)
+
+ @property
+ def matrix_html(self):
+ return self.df.applymap(lambda x: raw_value_to_html(x.raw_value))
+
+ @property
+ def matrix_layout(self):
+ return self.df.applymap(lambda x: x.layout)
+
+ @property
+ def matrix_gold_tags(self):
+ return self.df.applymap(lambda x: x.gold_tags)
+
+ # todo: remove gold_tags
+ @property
+ def matrix_tags(self):
+ return self.matrix_gold_tags
+
+ @classmethod
+ def from_file(cls, path, metadata, annotations=None, migrate=False, match_name=None, guessed_tags=None):
+ path = Path(path)
+ filename = path / metadata['filename']
+ df = read_str_csv(filename)
+ if 'layout' in metadata:
+ layout = read_str_csv(path / metadata['layout'])
+ else:
+ layout = None
+ if annotations is not None:
+ if not migrate:
+ # TODO: remove parser after migration is fully finished
+ table_ann = annotations.table_set.filter(name=metadata['filename'], parser="latexml") + [None]
+ table_ann = table_ann[0]
+ elif match_name is not None:
+ table_ann = annotations.table_set.filter(name=match_name) + [None]
+ table_ann = table_ann[0]
+ else:
+ table_ann = None
+ else:
+ table_ann = None
+ return cls(metadata['filename'], df, layout, metadata.get('caption'), metadata.get('figure_id'), table_ann, migrate, match_name, guessed_tags)
+
+ def _repr_html_(self):
+ return table_to_html(self.matrix_html.values, self.matrix_tags.values, self.matrix_layout.values)
+
+ def display(self):
+ display_html(self._repr_html_())
+
+ def _save_df(self, df, filename):
+ df.to_csv(filename, header=None, index=None)
+
+ def save(self, path, table_name, layout_name):
+ path = Path(path)
+ self._save_df(self.df.applymap(lambda x: x.raw_value), path / table_name)
+ self._save_df(self.df.applymap(lambda x: x.layout), path / layout_name)
+
+#####
+# this code is used to migrate table annotations from
+# tables parsed by htlatex to tables parsed by
+# latexml. After all annotated tables will be successfully
+# migrated, we switch back to match-by-name
+
+from unidecode import unidecode
+import string
+from collections import Counter
+
+figure_prefix_re = re.compile('^(table|figure)\s+([0-9]+|[ivxl]+)?')
+punctuation_table = str.maketrans('', '', string.punctuation)
+def normalize_string(s):
+ if s is None:
+ return ""
+
+ s = s.strip().lower()
+ s = figure_prefix_re.sub('', s).strip()
+ return unidecode(s.replace('\xa0', '').replace(' ', '')).translate(punctuation_table)
+
+def _remove_almost_empty_values(d):
+ return {k:v for k,v in d.items() if len(v) >= 10}
+
+def _keep_unique_values(d):
+ c = Counter(d.values())
+ unique = [k for k,v in c.items() if v == 1]
+ return {k: v for k,v in d.items() if v in unique}
+
+def _match_tables_by_captions(annotations, metadata):
+ if annotations is None:
+ return {}
+ old_captions = {x.name: normalize_string(x.desc) for x in annotations.table_set}
+ new_captions = {m['filename']: normalize_string(m['caption']) for m in metadata}
+ old_captions = _keep_unique_values(_remove_almost_empty_values(old_captions))
+ new_captions = _keep_unique_values(_remove_almost_empty_values(new_captions))
+ old_captions_reverse = {v:k for k,v in old_captions.items()}
+ return {new_name:old_captions_reverse[caption] for new_name, caption in new_captions.items() if caption in old_captions_reverse}
+
+def normalize_cell(s):
+ #s = reference_re.sub(' [] ', s)
+ return normalize_string(s)
+
+# begin of guess annotations mapping
+def create_cell_contexts(df):
+ cell_context = df.values
+ cells = np.pad(cell_context, 1, mode='constant', constant_values='')
+
+ slices = [slice(None, -2), slice(1,-1), slice(2, None)]
+
+ row_context = np.stack([cells[1:-1, s] for s in slices], axis=-1)
+ col_context = np.stack([cells[s, 1:-1] for s in slices], axis=-1)
+ box_context = np.stack([cells[s1, s2] for s1 in slices for s2 in slices], axis=-1)
+ return box_context, row_context, col_context, cell_context[...,None]
+
+def map_context(context, values):
+ ctx_len = context.shape[-1]
+ mapping = {}
+ for ctx, val in zip(context.reshape((-1, ctx_len)), values.reshape(-1)):
+ mapping.setdefault(tuple(ctx), set()).add(val)
+ return mapping
+
+REANNOTATE_TAG = 'reannotate'
+
+def guess_annotations(old_table, gold_tags, new_table):
+ df = pd.DataFrame().reindex_like(new_table).fillna(REANNOTATE_TAG)
+ if old_table.empty:
+ return 0, df
+ old_contexts = create_cell_contexts(old_table)
+ old_mappings = [map_context(ctx, gold_tags.values) for ctx in old_contexts]
+ new_contexts = create_cell_contexts(new_table)
+
+ rows, cols = new_table.shape
+ matched = 0
+ for row in range(rows):
+ for col in range(cols):
+ for mapping, context in zip(old_mappings, new_contexts):
+ ctx = tuple(context[row, col])
+ values = mapping.get(ctx, set())
+ if len(values) == 1:
+ (val,) = values
+ df.iloc[row, col] = val
+ matched += 1
+ break
+ return matched, df
+
+# end of guess annotations mapping
+
+
+def same_table(old_table, new_table):
+ return old_table.equals(new_table)
+
+DEB_PAPER="1607.00036v2"
+
+def deb(path, old_name, old_table, new_name, new_table):
+ if path.name == DEB_PAPER and old_name == "table_02.csv" == new_name:
+ print(old_table)
+ print(new_table)
+
+def _match_tables_by_content(path, annotations, metadata):
+ if annotations is None:
+ return {}, {}
+ old_tables = {x.name: (pd.DataFrame(x.matrix).applymap(normalize_cell), pd.DataFrame(x.matrix_gold_tags)) for x in annotations.table_set}
+ new_tables = {m['filename']: Table.from_file(path, m, None, None).df.applymap(lambda c: normalize_cell(c.value)) for m in metadata}
+ matched = {}
+ new_tags = {}
+ for new_name, new_table in new_tables.items():
+ max_hits = 0
+ matched_name = None
+ size = np.prod(new_table.shape)
+ guessed_tags = None
+ for old_name, (old_table, gold_tags) in old_tables.items():
+ hits, tags = guess_annotations(old_table, gold_tags, new_table)
+ if hits > max_hits:
+ max_hits = hits
+ matched_name = old_name
+ guessed_tags = tags
+ if max_hits > size / 2:
+ matched[new_name] = matched_name
+ new_tags[new_name] = guessed_tags
+ #deb(path, old_name, old_table, new_name, new_table)
+ #if same_table(old_table, new_table):
+ # if new_name in matched:
+ # print(f"Multiple matches for {path}/{new_name}: {matched[new_name]}, {old_name}")
+ # else:
+ # matched[new_name] = old_name
+ return matched, new_tags
+####
+
+def read_tables(path, annotations, migrate=False):
+ path = Path(path)
+ with open(path / "metadata.json", "r") as f:
+ metadata = json.load(f)
+
+ if migrate:
+ _matched_names_by_captions = {} #_match_tables_by_captions(annotations, metadata)
+ _matched_names_by_content, _guessed_tags = _match_tables_by_content(path, annotations, metadata)
+ _matched_names = _matched_names_by_captions
+ for new_name, old_name in _matched_names_by_content.items():
+ if new_name in _matched_names and _matched_names[new_name] != old_name:
+ print(f"Multiple matches for table {path}/{new_name}: {_matched_names[new_name]} by caption and {old_name} by content")
+ else:
+ _matched_names[new_name] = old_name
+ else:
+ _matched_names = {}
+ _guessed_tags = {}
+ return [Table.from_file(path, m, annotations, migrate=migrate, match_name=_matched_names.get(m["filename"]), guessed_tags=_guessed_tags.get(m["filename"])) for m in metadata]
diff --git a/axcell/errors.py b/axcell/errors.py
new file mode 100644
index 0000000..bcfa3ea
--- /dev/null
+++ b/axcell/errors.py
@@ -0,0 +1,12 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+class PipelineError(Exception):
+ pass
+
+
+class UnpackError(PipelineError):
+ pass
+
+
+class LatexConversionError(PipelineError):
+ pass
diff --git a/axcell/helpers/__init__.py b/axcell/helpers/__init__.py
new file mode 100644
index 0000000..c82eacb
--- /dev/null
+++ b/axcell/helpers/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+from .latex_converter import LatexConverter
+from .unpack import Unpack
+
+__all__ = ["LatexConverter", "Unpack"]
diff --git a/axcell/helpers/cache.py b/axcell/helpers/cache.py
new file mode 100644
index 0000000..1c7dfda
--- /dev/null
+++ b/axcell/helpers/cache.py
@@ -0,0 +1,61 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+import pandas as pd
+import json
+from collections import defaultdict
+from pathlib import Path
+
+
+# these functions are used to cache various results
+# of corresponding pipeline steps, to make it faster to
+# rerun the pipeline or run in on batch of papers with various
+# steps on different machines. The exchange formats are ad hoc and
+# can be changed.
+
+
+def _load_json(path):
+ with Path(path).open('rt') as f:
+ return json.load(f)
+
+
+def _save_json(obj, path):
+ with Path(path).open('wt') as f:
+ json.dump(obj, f)
+
+
+def load_references(path):
+ return _load_json(path)
+
+
+def save_references(references, path):
+ _save_json(references, path)
+
+
+def load_tags(path):
+ return _load_json(path)
+
+
+def save_tags(tags, path):
+ _save_json(tags, path)
+
+
+def load_structure(path):
+ return _load_json(path)
+
+
+def save_structure(structure, path):
+ _save_json(structure, path)
+
+
+def load_proposals(path):
+ dtypes = defaultdict(lambda: str)
+ dtypes['confidence'] = float
+ dtypes['parsed'] = float
+
+ na_values = {'confidence': '', 'parsed': ''}
+ proposals = pd.read_csv(path, index_col=0, dtype=dtypes, na_values=na_values, keep_default_na=False)
+ return proposals
+
+
+def save_proposals(proposals, path):
+ proposals.to_csv(path)
diff --git a/axcell/helpers/datasets.py b/axcell/helpers/datasets.py
new file mode 100644
index 0000000..9e89e56
--- /dev/null
+++ b/axcell/helpers/datasets.py
@@ -0,0 +1,11 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+import pandas as pd
+
+
+def read_arxiv_papers(path):
+ return pd.read_csv(path)
+
+
+def read_tables_annotations(path):
+ return pd.read_json(path)
diff --git a/axcell/helpers/evaluate.py b/axcell/helpers/evaluate.py
new file mode 100644
index 0000000..738c620
--- /dev/null
+++ b/axcell/helpers/evaluate.py
@@ -0,0 +1,99 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+import re
+import pandas as pd
+
+from axcell.data.paper_collection import remove_arxiv_version
+
+
+def norm_score_str(x):
+ x = str(x)
+ if re.match('^(\+|-|)(\d+)\.9{5,}$', x):
+ x = re.sub('^(\+|-|)(\d+)\.9{5,}$', lambda a: a.group(1)+str(int(a.group(2))+1), x)
+ elif x.endswith('9' * 5) and '.' in x:
+ x = re.sub(r'([0-8])9+$', lambda a: str(int(a.group(1))+1), x)
+ if '.' in x:
+ x = re.sub(r'0+$', '', x)
+ if x[-1] == '.':
+ x = x[:-1]
+ if x == '-0':
+ x = '0'
+ return x
+
+
+epsilon = 1e-10
+
+
+def precision(tp, fp):
+ pred_positives = tp + fp + epsilon
+ return ((1.0 * tp) / pred_positives)
+
+
+def recall(tp, fn):
+ true_positives = tp + fn + epsilon
+ return ((1.0 * tp) / true_positives)
+
+
+def f1(prec, recall):
+ norm = prec + recall + epsilon
+ return (2 * prec * recall / norm)
+
+
+def stats(predictions, ground_truth, axis=None):
+ gold = pd.DataFrame(ground_truth, columns=["paper", "task", "dataset", "metric", "value"])
+ pred = pd.DataFrame(predictions, columns=["paper", "task", "dataset", "metric", "value"])
+
+ if axis == 'tdm':
+ columns = ['paper', 'task', 'dataset', 'metric']
+ elif axis == 'tdms' or axis is None:
+ columns = ['paper', 'task', 'dataset', 'metric', 'value']
+ else:
+ columns = ['paper', axis]
+ gold = gold[columns].drop_duplicates()
+ pred = pred[columns].drop_duplicates()
+
+ results = gold.merge(pred, on=columns, how="outer", indicator=True)
+
+ is_correct = results["_merge"] == "both"
+ no_pred = results["_merge"] == "left_only"
+ no_gold = results["_merge"] == "right_only"
+
+ results["TP"] = is_correct.astype('int8')
+ results["FP"] = no_gold.astype('int8')
+ results["FN"] = no_pred.astype('int8')
+
+ m = results.groupby(["paper"]).agg({"TP": "sum", "FP": "sum", "FN": "sum"})
+ m["precision"] = precision(m.TP, m.FP)
+ m["recall"] = recall(m.TP, m.FN)
+ m["f1"] = f1(m.precision, m.recall)
+
+ TP_ALL = m.TP.sum()
+ FP_ALL = m.FP.sum()
+ FN_ALL = m.FN.sum()
+
+ prec, reca = precision(TP_ALL, FP_ALL), recall(TP_ALL, FN_ALL)
+ return {
+ 'Micro Precision': prec,
+ 'Micro Recall': reca,
+ 'Micro F1': f1(prec, reca),
+ 'Macro Precision': m.precision.mean(),
+ 'Macro Recall': m.recall.mean(),
+ 'Macro F1': m.f1.mean()
+ }
+
+
+def evaluate(predictions, ground_truth):
+ predictions = predictions.copy()
+ ground_truth = ground_truth.copy()
+ predictions['value'] = predictions['score' if 'score' in predictions else 'value'].apply(norm_score_str)
+ ground_truth['value'] = ground_truth['score' if 'score' in ground_truth else 'value'].apply(norm_score_str)
+ predictions['paper'] = predictions['arxiv_id'].apply(remove_arxiv_version)
+ ground_truth['paper'] = ground_truth['arxiv_id'].apply(remove_arxiv_version)
+
+ metrics = []
+ for axis in [None, "tdm", "task", "dataset", "metric"]:
+ s = stats(predictions, ground_truth, axis)
+ s['type'] = {'tdms': 'TDMS', 'tdm': 'TDM', 'task': 'Task', 'dataset': 'Dataset', 'metric': 'Metric'}.get(axis)
+ metrics.append(s)
+ columns = ['Micro Precision', 'Micro Recall', 'Micro F1', 'Macro Precision', 'Macro Recall', 'Macro F1']
+ return pd.DataFrame(metrics, columns=columns)
diff --git a/axcell/helpers/explainers.py b/axcell/helpers/explainers.py
new file mode 100644
index 0000000..a88636a
--- /dev/null
+++ b/axcell/helpers/explainers.py
@@ -0,0 +1,218 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+from axcell.models.linking.metrics import Metrics
+from ..models.structure import TableType
+from ..loggers import StructurePredictionEvaluator, LinkerEvaluator, FilteringEvaluator
+import pandas as pd
+import numpy as np
+from ..helpers.jupyter import table_to_html
+from axcell.models.linking.format import extract_value
+from axcell.helpers.optimize import optimize_filters
+
+
+class Reason:
+ pass
+
+
+class IrrelevantTable(Reason):
+ def __init__(self, paper, table, table_type, probs):
+ self.paper = paper
+ self.table = table
+ self.table_type = table_type
+ self.probs = pd.DataFrame(probs, columns=["type", "probability"])
+
+ def __str__(self):
+ return f"Table {self.table.name} was labelled as {self.table_type.name}."
+
+ def _repr_html_(self):
+ prediction = f'{self}
'
+ caption = f'Caption: {self.table.caption}
'
+ probs = self.probs.style.format({"probability": "{:.2f}"})._repr_html_()
+ return prediction + caption + probs
+
+
+class MislabeledCell(Reason):
+ def __init__(self, paper, table, row, col, probs):
+ self.paper = paper
+ self.table = table
+
+
+class TableExplanation:
+ def __init__(self, paper, table, table_type, proposals, reasons, topk):
+ self.paper = paper
+ self.table = table
+ self.table_type = table_type
+ self.proposals = proposals
+ self.reasons = reasons
+ self.topk = topk
+
+ def _format_tooltip(self, proposal):
+ return f"dataset: {proposal.dataset}\n" \
+ f"metric: {proposal.metric}\n" \
+ f"task: {proposal.task}\n" \
+ f"score: {proposal.parsed}\n" \
+ f"confidence: {proposal.confidence:0.2f}"
+
+ def _format_topk(self, topk):
+ return ""
+
+ def _repr_html_(self):
+ matrix = self.table.matrix_html.values
+ predictions = np.zeros_like(matrix, dtype=object)
+ tooltips = np.zeros_like(matrix, dtype=object)
+ for cell_ext_id, proposal in self.proposals.iterrows():
+ paper_id, table_name, rc = cell_ext_id.split("/")
+ row, col = [int(x) for x in rc.split('.')]
+ if cell_ext_id in self.reasons:
+ reason = self.reasons[cell_ext_id]
+ tooltips[row, col] = reason
+ if reason.startswith("replaced by "):
+ tooltips[row, col] += "\n\n" + self._format_tooltip(proposal)
+ elif reason.startswith("confidence "):
+ tooltips[row, col] += "\n\n" + self._format_topk(self.topk[row, col])
+ else:
+ predictions[row, col] = 'final-proposal'
+ tooltips[row, col] = self._format_tooltip(proposal)
+
+ table_type_html = f'Table {self.table.name} was labelled as {self.table_type.name}.
'
+ caption_html = f'Caption: {self.table.caption}
'
+ table_html = table_to_html(matrix,
+ self.table.matrix_tags.values,
+ self.table.matrix_layout.values,
+ predictions,
+ tooltips)
+ html = table_type_html + caption_html + table_html
+ proposals = self.proposals[~self.proposals.index.isin(self.reasons.index)]
+ if len(proposals):
+ proposals = proposals[["dataset", "metric", "task", "model", "parsed"]]\
+ .reset_index(drop=True).rename(columns={"parsed": "score"})
+ html2 = proposals._repr_html_()
+ return f""
+ return html
+
+
+class Explainer:
+ _sota_record_columns = ['task', 'dataset', 'metric', 'format', 'model', 'model_type', 'raw_value', 'parsed']
+
+ def __init__(self, pipeline_logger, paper_collection, gold_sota_records=None):
+ self.paper_collection = paper_collection
+ self.gold_sota_records = gold_sota_records
+ self.spe = StructurePredictionEvaluator(pipeline_logger, paper_collection)
+ self.le = LinkerEvaluator(pipeline_logger)
+ self.fe = FilteringEvaluator(pipeline_logger)
+
+ def explain(self, paper, cell_ext_id):
+ paper_id, table_name, rc = cell_ext_id.split('/')
+ if paper.paper_id != paper_id:
+ return "No such cell"
+
+ table_type, probs = self.spe.get_table_type_predictions(paper_id, table_name)
+
+ if table_type == TableType.IRRELEVANT:
+ return IrrelevantTable(paper, paper.table_by_name(table_name), table_type, probs)
+
+ all_proposals = self.le.proposals[paper_id]
+ reasons = self.fe.reason
+ table_ext_id = f"{paper_id}/{table_name}"
+ table_proposals = all_proposals[all_proposals.index.str.startswith(table_ext_id+"/")]
+ topk = {(row, col): topk for (pid, tn, row, col), topk in self.le.topk.items()
+ if (pid, tn) == (paper_id, table_name)}
+
+ return TableExplanation(paper, paper.table_by_name(table_name), table_type, table_proposals, reasons, topk)
+
+ row, col = [int(x) for x in rc.split('.')]
+
+ reason = self.fe.reason.get(cell_ext_id)
+ if reason is None:
+ pass
+ else:
+ return reason
+
+ def _get_table_sota_records(self, table):
+
+ first_model = lambda x: ([a for a in x if a.startswith('model')] + [''])[0]
+ if len(table.sota_records):
+ matrix = table.matrix.values
+ tags = table.matrix_tags
+ model_type_col = tags.apply(first_model)
+ model_type_row = tags.T.apply(first_model)
+ sota_records = table.sota_records.copy()
+ sota_records['model_type'] = ''
+ sota_records['raw_value'] = ''
+ for cell_ext_id, record in sota_records.iterrows():
+ name, rc = cell_ext_id.split('/')
+ row, col = [int(x) for x in rc.split('.')]
+ record.model_type = model_type_col[col] or model_type_row[row]
+ record.raw_value = matrix[row, col]
+
+ sota_records["parsed"] = sota_records[["raw_value", "format"]].apply(
+ lambda row: float(extract_value(row.raw_value, row.format)), axis=1)
+
+ sota_records = sota_records[sota_records["parsed"] == sota_records["parsed"]]
+
+ strip_cols = ["task", "dataset", "format", "metric", "raw_value", "model", "model_type"]
+ sota_records = sota_records.transform(
+ lambda x: x.str.strip() if x.name in strip_cols else x)
+ return sota_records[self._sota_record_columns]
+ else:
+ empty = pd.DataFrame(columns=self._sota_record_columns)
+ empty.index.rename("cell_ext_id", inplace=True)
+ return empty
+
+ def _get_sota_records(self, paper):
+ if not len(paper.tables):
+ empty = pd.DataFrame(columns=self._sota_record_columns)
+ empty.index.rename("cell_ext_id", inplace=True)
+ return empty
+ records = [self._get_table_sota_records(table) for table in paper.tables]
+ records = pd.concat(records)
+ records.index = paper.paper_id + "/" + records.index
+ records.index.rename("cell_ext_id", inplace=True)
+ return records
+
+ def linking_metrics(self, experiment_name="unk", topk_metrics=False, filtered=True, confidence=0.0):
+ paper_ids = list(self.le.proposals.keys())
+
+ proposals = pd.concat(self.le.proposals.values())
+
+ # if not topk_metrics:
+ if filtered:
+ proposals = proposals[~proposals.index.isin(self.fe.reason.index)]
+ if confidence:
+ proposals = proposals[proposals.confidence > confidence]
+
+ papers = {paper_id: self.paper_collection.get_by_id(paper_id) for paper_id in paper_ids}
+ missing = [paper_id for paper_id, paper in papers.items() if paper is None]
+ if missing:
+ print("Missing papers in paper collection:")
+ print(", ".join(missing))
+ papers = [paper for paper in papers.values() if paper is not None]
+
+ # if not len(papers):
+ # gold_sota_records = pd.DataFrame(columns=self._sota_record_columns)
+ # gold_sota_records.index.rename("cell_ext_id", inplace=True)
+ # else:
+ # gold_sota_records = pd.concat([self._get_sota_records(paper) for paper in papers])
+ if self.gold_sota_records is None:
+ gold_sota_records = pd.DataFrame(columns=self._sota_record_columns)
+ gold_sota_records.index.rename("cell_ext_id", inplace=True)
+ else:
+
+ gold_sota_records = self.gold_sota_records
+ which = gold_sota_records.index.to_series().str.split("/", expand=True)[0]\
+ .isin([paper.paper_id for paper in papers])
+ gold_sota_records = gold_sota_records[which]
+
+ df = gold_sota_records.merge(proposals, 'outer', left_index=True, right_index=True, suffixes=['_gold', '_pred'])
+ df = df.reindex(sorted(df.columns), axis=1)
+ df = df.fillna('not-present')
+ if "experiment_name" in df.columns:
+ del df["experiment_name"]
+
+ metrics = Metrics(df, experiment_name=experiment_name, topk_metrics=topk_metrics)
+ return metrics
+
+
+ def optimize_filters(self, metrics_info):
+ results = optimize_filters(self, metrics_info)
+ return results
diff --git a/axcell/helpers/interpret.py b/axcell/helpers/interpret.py
new file mode 100644
index 0000000..396ebe8
--- /dev/null
+++ b/axcell/helpers/interpret.py
@@ -0,0 +1,61 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+from fastai.text.interpret import TextClassificationInterpretation as AbsTextClassificationInterpretation, _eval_dropouts
+from fastai.basic_data import DatasetType
+import torch
+
+
+__all__ = ["TextClassificationInterpretation", "TextMultiClassificationInterpretation"]
+
+
+class TextClassificationInterpretation(AbsTextClassificationInterpretation):
+ @classmethod
+ def from_learner(cls, learner):
+ empty_preds = torch.Tensor([[1]])
+ return cls(learner, empty_preds, None, None)
+
+ def intrinsic_attention(self, text:str, class_id:int=None):
+ """Calculate the intrinsic attention of the input w.r.t to an output `class_id`, or the classification given by the model if `None`.
+ Similar as in base class, but does not apply abs() before summing gradients.
+ """
+ self.model.train()
+ _eval_dropouts(self.model)
+ self.model.zero_grad()
+ self.model.reset()
+ ids = self.data.one_item(text)[0]
+ emb = self.model[0].module.encoder(ids).detach().requires_grad_(True)
+ lstm_output = self.model[0].module(emb, from_embeddings=True)
+ self.model.eval()
+ cl = self.model[1](lstm_output + (torch.zeros_like(ids).byte(),))[0].softmax(dim=-1)
+ if class_id is None: class_id = cl.argmax()
+ cl[0][class_id].backward()
+ # attn = emb.grad.squeeze().abs().sum(dim=-1)
+ # attn /= attn.max()
+ attn = emb.grad.squeeze().sum(dim=-1)
+ attn = attn / attn.abs().max() * 0.5 + 0.5
+ tokens = self.data.single_ds.reconstruct(ids[0])
+ return tokens, attn
+
+
+class TextMultiClassificationInterpretation(TextClassificationInterpretation):
+ def intrinsic_attention(self, text:str, class_id:int=None):
+ """Calculate the intrinsic attention of the input w.r.t to an output `class_id`, or the classification given by the model if `None`.
+ Similar as in base class, but uses sigmoid instead of softmax and does not apply abs() before summing gradients.
+ """
+ self.model.train()
+ _eval_dropouts(self.model)
+ self.model.zero_grad()
+ self.model.reset()
+ ids = self.data.one_item(text)[0]
+ emb = self.model[0].module.encoder(ids).detach().requires_grad_(True)
+ lstm_output = self.model[0].module(emb, from_embeddings=True)
+ self.model.eval()
+ cl = self.model[1](lstm_output + (torch.zeros_like(ids).byte(),))[0].sigmoid()
+ if class_id is None: class_id = cl.argmax()
+ cl[0][class_id].backward()
+ # attn = emb.grad.squeeze().abs().sum(dim=-1)
+ # attn /= attn.max()
+ attn = emb.grad.squeeze().sum(dim=-1)
+ attn = attn / attn.abs().max() * 0.5 + 0.5
+ tokens = self.data.single_ds.reconstruct(ids[0])
+ return tokens, attn
diff --git a/sota_extractor2/helpers/jupyter.py b/axcell/helpers/jupyter.py
similarity index 51%
rename from sota_extractor2/helpers/jupyter.py
rename to axcell/helpers/jupyter.py
index 29d3e81..6409a09 100644
--- a/sota_extractor2/helpers/jupyter.py
+++ b/axcell/helpers/jupyter.py
@@ -1,5 +1,10 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
from IPython.core.display import display, HTML
from .table_style import table_style
+import numpy as np
+
+
def set_seed(seed, name):
import torch
import numpy as np
@@ -9,11 +14,11 @@ def set_seed(seed, name):
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
-def display_html(s): return display(HTML(s))
+def display_html(s): return display(HTML(s))
-def display_table(table, structure=None):
+def table_to_html(table, structure=None, layout=None, predictions=None, tooltips=None):
"""
matrix - 2d ndarray with cell values
strucutre - 2d ndarray with structure annotation
@@ -23,15 +28,23 @@ def display_table(table, structure=None):
else:
matrix = table
if structure is None: structure = table.matrix_gold_tags
+ if layout is None: layout = np.zeros_like(matrix, dtype=str)
+ if predictions is None: predictions = np.zeros_like(matrix, dtype=str)
+ if tooltips is None: tooltips = np.zeros_like(matrix, dtype=str)
html = []
html.append(table_style)
html.append('')
html.append("
")
- for row,struc_row in zip(matrix, structure):
+ for row,struc_row, layout_row, preds_row, tt_row in zip(matrix, structure, layout, predictions, tooltips):
html.append("")
- for cell,struct in zip(row,struc_row):
- html.append(f'| {cell} | ')
+ for cell,struct,layout,preds, tt in zip(row,struc_row,layout_row,preds_row, tt_row):
+ html.append(f'{cell} | ')
html.append("
")
html.append("
")
html.append('