forked from bazingagin/npc_gzip
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
removed unused imports remove unused commented code
- Loading branch information
Zach Bloss
committed
Jul 17, 2023
1 parent
b51c218
commit 55792af
Showing
1 changed file
with
23 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,42 +1,40 @@ | ||
# Compressor Framework | ||
import gzip | ||
import bz2 | ||
import gzip | ||
import lzma | ||
# from PIL.PngImagePlugin import getchunks | ||
# from PIL import Image | ||
import sys | ||
from tqdm import tqdm | ||
import torch.nn.functional as F | ||
|
||
|
||
import io | ||
|
||
|
||
class DefaultCompressor: | ||
"""for non-neural-based compressor""" | ||
def __init__(self, compressor, typ='text'): | ||
if compressor == 'gzip': | ||
|
||
def __init__(self, compressor: str, typ: str = "text"): | ||
if compressor == "gzip": | ||
self.compressor = gzip | ||
elif compressor == 'bz2': | ||
elif compressor == "bz2": | ||
self.compressor = bz2 | ||
elif compressor == 'lzma': | ||
elif compressor == "lzma": | ||
self.compressor = lzma | ||
else: | ||
raise RuntimeError("Unsupported compressor") | ||
self.type = typ | ||
def get_compressed_len(self, x): | ||
if self.type == 'text': | ||
return len(self.compressor.compress(x.encode('utf-8'))) | ||
|
||
def get_compressed_len(self, x: str) -> int: | ||
""" | ||
Calculates the size of `x` once compressed. | ||
""" | ||
if self.type == "text": | ||
return len(self.compressor.compress(x.encode("utf-8"))) | ||
else: | ||
return len(self.compressor.compress(np.array(x).tobytes())) | ||
def get_bits_per_char(self, original_fn): | ||
with open(original_fn) as fo: | ||
data = fo.read() | ||
compressed_str = self.compressor.compress(data.encode('utf-8')) | ||
return len(compressed_str)*8/len(data) | ||
|
||
def get_bits_per_char(self, original_fn: str) -> float: | ||
""" | ||
Returns the compressed size of the original function | ||
in bits. | ||
|
||
"""Test Compressors""" | ||
# comp = DefaultCompressor('gzip') | ||
# print(comp.get_compressed_len('Hello world')) | ||
""" | ||
with open(original_fn) as fo: | ||
data = fo.read() | ||
compressed_str = self.compressor.compress(data.encode("utf-8")) | ||
return len(compressed_str) * 8 / len(data) |