forked from explosion/thinc
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsetup.py
2736 lines (2333 loc) · 97.1 KB
/
setup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python
from __future__ import print_function
import io
import os.path
import subprocess
import sys
import contextlib
from distutils.command.build_ext import build_ext
from distutils.sysconfig import get_python_inc
from distutils import ccompiler, msvccompiler
from distutils.ccompiler import new_compiler
from setuptools import Extension, setup
PACKAGES = [
'thinc',
'thinc.tests',
'thinc.tests.unit',
'thinc.tests.integration',
'thinc.tests.linear',
'thinc.linear',
'thinc.neural',
'thinc.extra',
'thinc.neural._classes',
'thinc.extra._vendorized'
]
MOD_NAMES = [
'thinc.linalg',
'thinc.openblas',
'thinc.structs',
'thinc.typedefs',
'thinc.linear.avgtron',
'thinc.linear.features',
'thinc.linear.serialize',
'thinc.linear.sparse',
'thinc.linear.linear',
'thinc.neural.optimizers',
'thinc.neural.ops',
'thinc.neural.gpu_ops',
'thinc.neural._aligned_alloc',
'thinc.neural._fast_maxout_cnn',
'thinc.extra.eg',
'thinc.extra.mb',
'thinc.extra.search',
'thinc.extra.cache',
]
compile_options = {'msvc' : ['/Ox', '/EHsc'],
'other' : {
'gcc': ['-O2', '-Wno-strict-prototypes', '-Wno-unused-function'],
'nvcc': ['-arch=sm_30', '--ptxas-options=-v', '-c', '--compiler-options', "'-fPIC'"]}}
link_options = {'msvc' : [], 'other' : []}
class Openblas(Extension):
cpu_info = None
@property
def is_haswell(self):
if self.cpu_info is None:
self.cpu_info = get_cpu_info()
return self.cpu_info['model'] >= 63
def build_objects(self, compiler, src_dir):
if compiler.platform != 'nt':
c_flags = list(compiler.compiler)
compiler.compiler = compiler.compiler[:1] + ['-fPIC']
cso_flags = list(compiler.compiler_so)
compiler.compiler_so = compiler.compiler_so[:1] + ['-fPIC']
pre_flags = list(compiler.preprocessor)
compiler.preprocessor = compiler.preprocessor[:1] + ['-fPIC']
compiler.src_extensions.append('.S')
if hasattr(compiler, '_c_extensions'):
compiler._c_extensions.append('.S')
self.include_dirs.append(src_dir)
objects = []
for iface in ['gemm', 'axpy']:
objects.extend(self.compile_interface(
compiler, src_dir, 'cblas_s%s' % iface, iface))
objects.extend(self.build_gemm(compiler, src_dir))
objects.extend(self.build_level1(compiler, src_dir))
for other in ['parameter', 'memory', 'init', 'openblas_env', 'xerbla']:
objects.extend(self.compile_driver(compiler,
os.path.join(src_dir, 'driver', 'others'),
other, '%s.c' % other, []))
self.extra_objects.extend(objects)
self.extra_link_args.append('-Wl,--no-undefined')
if compiler.platform != 'nt':
compiler.compiler = c_flags
compiler.compiler_so = cso_flags
compiler.preprocessor = pre_flags
return objects
def build_gemm(self, compiler, src_dir):
objects = []
for flavor in ['nn', 'nt', 'tn', 'tt']:
name = 'sgemm_%s' % flavor
objects.extend(
self.compile_driver(
compiler, os.path.join(src_dir, 'driver', 'level3'),
name, 'gemm.c', [(flavor.upper(), None)]))
if self.is_haswell:
objects.extend(
self.compile_driver(compiler, os.path.join(src_dir, 'kernel', 'x86_64'),
'sgemm_kernel', 'sgemm_kernel_16x4_haswell.S', []))
else:
objects.extend(
self.compile_driver(compiler, os.path.join(src_dir, 'kernel', 'x86_64'),
'sgemm_kernel', 'sgemm_kernel_16x4_sandy.S', []))
objects.extend(
self.compile_driver(
compiler, os.path.join(src_dir, 'kernel', 'x86_64'),
'sgemm_beta', 'gemm_beta.S', []))
objects.extend(
self.compile_driver(
compiler, os.path.join(src_dir, 'kernel', 'generic'),
'sgemm_itcopy', 'gemm_tcopy_16.c', []))
objects.extend(
self.compile_driver(
compiler, os.path.join(src_dir, 'kernel', 'generic'),
'sgemm_incopy', 'gemm_ncopy_16.c', []))
objects.extend(
self.compile_driver(
compiler, os.path.join(src_dir, 'kernel', 'generic'),
'sgemm_oncopy', 'gemm_ncopy_4.c', []))
objects.extend(
self.compile_driver(
compiler, os.path.join(src_dir, 'kernel', 'generic'),
'sgemm_otcopy', 'gemm_tcopy_4.c', []))
return objects
def build_level1(self, compiler, src_dir):
objects = []
objects.extend(self.compile_driver(compiler,
os.path.join(src_dir, 'kernel', 'x86_64'),
'saxpy_k', 'saxpy.c', []))
return objects
def compile_driver(self, compiler, src_dir, name, src_name, macros):
if compiler.platform == 'nt':
macros.append(('OS_WINDOWS', None))
macros.append(('C_MSVC', None))
macros.append(('WINDOWS_ABI', None))
args = []
else:
macros.append(('OS_LINUX', None))
args = ['-c', '-O2', '-Wall', '-m64', '-fPIC']
# Architecture
if self.is_haswell:
macros.append(('HASWELL', None))
else:
macros.append(('SANDYBRIDGE', None))
macros.append(('USE_OPENMP', 1))
# Stuff we're not building
macros.append(('F_INTERFACE_GFORT', None))
macros.append(('NO_LAPACK', None))
macros.append(('NO_LAPACKE', None))
macros.append(('DOUBLE',))
macros.append(('COMPLEX',))
# Settings that maybe matter for optimization?
macros.append(('MAX_STACK_ALLOC', '2048'))
macros.append(('NO_WARMUP',))
macros.append(('MAX_CPU_NUMBER', '4'))
# Fill in the template
macros.append(('ASMNAME', name))
macros.append(('ASMFNAME', '%s_' % name))
macros.append(('NAME', '%s_' % name))
macros.append(('CNAME', name))
macros.append(('CHAR_NAME', "%s_" % name))
macros.append(('CHAR_CNAME', "%s_" % name))
src = os.path.join(src_dir, src_name)
obj = compiler.compile([src], output_dir=src_dir,
macros=macros, include_dirs=self.include_dirs,
extra_postargs=args)
output = os.path.join(src_dir, name+'.' + obj[0].split('.')[-1])
if os.path.exists(output):
os.unlink(output)
compiler.move_file(obj[0], output)
return [output]
def compile_interface(self, compiler, src_dir, name, src_name):
macros = []
if compiler.platform == 'nt':
macros.append(('OS_WINDOWS', None))
macros.append(('WINDOWS_ABI', None))
macros.append(('C_MSVC', None))
else:
macros.append(('OS_LINUX', None))
# Architecture
if self.is_haswell:
macros.append(('HASWELL', None))
else:
macros.append(('SANDYBRIDGE', None))
macros.append(('USE_OPENMP', 1))
macros.append(('MAX_STACK_ALLOC', '2048'))
macros.append(('F_INTERFACE_GFORT', None))
macros.append(('NO_LAPACK', None))
macros.append(('NO_LAPACKE', None))
# Undefine. Fucking attrocious api..
macros.append(('DOUBLE',))
macros.append(('COMPLEX',))
macros.append(('NO_WARMUP',))
macros.append(('MAX_CPU_NUMBER', '4'))
macros.append(('ASMNAME', name))
macros.append(('ASMFNAME', '%s_' % name))
macros.append(('NAME', '%s_' % name))
macros.append(('CNAME', name))
macros.append(('CHAR_NAME', "%s_" % name))
macros.append(('CHAR_CNAME', name))
macros.append(('CBLAS', None))
src = os.path.join(src_dir, 'interface', src_name+'.c')
obj = compiler.compile([src], output_dir=src_dir,
macros=macros, include_dirs=self.include_dirs)
output = os.path.join(src_dir, name+'.' + obj[0].split('.')[-1])
if os.path.exists(output):
os.unlink(output)
compiler.move_file(obj[0], output)
return [output]
def customize_compiler_for_nvcc(self):
"""inject deep into distutils to customize how the dispatch
to gcc/nvcc works.
If you subclass UnixCCompiler, it's not trivial to get your subclass
injected in, and still have the right customizations (i.e.
distutils.sysconfig.customize_compiler) run on it. So instead of going
the OO route, I have this. Note, it's kindof like a wierd functional
subclassing going on."""
# tell the compiler it can processes .cu
self.src_extensions.append('.cu')
# save references to the default compiler_so and _comple methods
if hasattr(self, 'compiler_so'):
default_compiler_so = self.compiler_so
else:
# This was put in for Windows, but I'm running blind here...
default_compiler_so = None
super = self._compile
# now redefine the _compile method. This gets executed for each
# object but distutils doesn't have the ability to change compilers
# based on source extension: we add it.
def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
if os.path.splitext(src)[1] == '.cu' and CUDA is not None:
# use the cuda for .cu files
if hasattr(self, 'set_executable'):
# This was put in for Windows, but I'm running blind here...
self.set_executable('compiler_so', CUDA['nvcc'])
# use only a subset of the extra_postargs, which are 1-1 translated
# from the extra_compile_args in the Extension class
postargs = extra_postargs['nvcc']
else:
postargs = extra_postargs['gcc']
super(obj, src, ext, cc_args, postargs, pp_opts)
# reset the default compiler_so, which we might have changed for cuda
self.compiler_so = default_compiler_so
# inject our redefined _compile method into the class
self._compile = _compile
# By subclassing build_extensions we have the actual compiler that will be used
# which is really known only after finalize_options
# http://stackoverflow.com/questions/724664/python-distutils-how-to-get-a-compiler-that-is-going-to-be-used
class build_ext_options:
def build_options(self):
src_dir = os.path.join(os.path.dirname(__file__), 'thinc', '_files')
if hasattr(self.compiler, 'initialize'):
self.compiler.initialize()
self.compiler.platform = sys.platform[:6]
for e in self.extensions:
if isinstance(e, Openblas):
if 'THINC_BLAS' in os.environ:
lib_loc = os.environ['THINC_BLAS']
lib_path, lib_name = os.path.split(lib_loc)
if lib_name.endswith(('.so', '.dylib')):
is_shared = True
lib_name = lib_name.rsplit('.', 1)[0][3:]
else:
is_shared = False
print('Using BLAS:', lib_path, lib_name)
compile_options['other']['gcc'].append('-L%s' % lib_path)
link_options['other'].append('-L%s' % lib_path)
if is_shared:
compile_options['other']['gcc'].append('-l%s' % lib_name)
link_options['other'].append('-l%s' % lib_name)
else:
compile_options['other']['gcc'].append('-l:%s' % lib_name)
link_options['other'].append('-l:%s' % lib_name)
link_options['msvc'].append(lib_loc)
elif self.compiler.platform == 'darwin':
e.extra_compile_args.append('-framework Accelerate')
e.extra_link_args.append('-framework Accelerate')
elif self.compiler.compiler_type == 'msvc':
clang = new_compiler(plat='nt', compiler='unix')
clang.platform = 'nt'
clang.compiler = [locate_windows_llvm()]
clang.compiler_so = clang.compiler
clang.library_dirs.extend(self.compiler.library_dirs)
clang.include_dirs = self.compiler.include_dirs
e.build_objects(clang, src_dir)
else:
e.build_objects(self.compiler, src_dir)
e.extra_compile_args = compile_options.get(
self.compiler.compiler_type, compile_options['other'])
e.extra_link_args = link_options.get(
self.compiler.compiler_type, link_options['other'])
class build_ext_subclass(build_ext, build_ext_options):
def build_extensions(self):
build_ext_options.build_options(self)
customize_compiler_for_nvcc(self.compiler)
build_ext.build_extensions(self)
def generate_cython(root, source):
print('Cythonizing sources')
p = subprocess.call([sys.executable,
os.path.join(root, 'bin', 'cythonize.py'),
source], env=os.environ)
if p != 0:
raise RuntimeError('Running cythonize failed')
def find_in_path(name, path):
"Find a file in a search path"
#adapted fom http://code.activestate.com/recipes/52224-find-a-file-given-a-search-path/
for dir in path.split(os.pathsep):
binpath = os.path.join(dir, name)
if os.path.exists(binpath):
return os.path.abspath(binpath)
return None
def locate_cuda():
"""Locate the CUDA environment on the system
Returns a dict with keys 'home', 'nvcc', 'include', and 'lib64'
and values giving the absolute path to each directory.
Starts by looking for the CUDAHOME env variable. If not found, everything
is based on finding 'nvcc' in the PATH.
"""
# first check if the CUDAHOME env variable is in use
if 'CUDA_HOME' in os.environ:
home = os.environ['CUDA_HOME']
nvcc = os.path.join(home, 'bin', 'nvcc')
elif os.path.exists('/usr/local/cuda/bin/nvcc'):
home = '/usr/local/cuda'
nvcc = os.path.join(home, 'bin', 'nvcc')
else:
# otherwise, search the PATH for NVCC
nvcc = find_in_path('nvcc', os.environ['PATH'])
if nvcc is None:
print('Warning: The nvcc binary could not be located in your $PATH. '
'For GPU capability, either add it to your path, or set $CUDA_HOME')
return None
home = os.path.dirname(os.path.dirname(nvcc))
cudaconfig = {'home':home, 'nvcc':nvcc,
'include': os.path.join(home, 'include'),
'lib64': os.path.join(home, 'lib64')}
for k, v in cudaconfig.items():
if not os.path.exists(v):
print('Warning: The CUDA %s path could not be located in %s' % (k, v))
return None
return cudaconfig
def locate_windows_llvm():
# first check if the LLVM_HOME env variable is in use
if 'LLVM_HOME' in os.environ:
home = os.environ['LLVM_HOME']
return os.path.join(home, 'bin', 'clang.exe')
else:
# otherwise, search the PATH for NVCC
clang = find_in_path('clang.exe', os.environ['PATH'])
if clang is None:
clang = r"C:\Program Files\LLVM\bin\clang.exe"
return clang
CUDA = locate_cuda()
def is_source_release(path):
return os.path.exists(os.path.join(path, 'PKG-INFO'))
def clean(path):
for name in MOD_NAMES:
name = name.replace('.', '/')
for ext in ['.so', '.html', '.cpp', '.c']:
file_path = os.path.join(path, name + ext)
if os.path.exists(file_path):
os.unlink(file_path)
@contextlib.contextmanager
def chdir(new_dir):
old_dir = os.getcwd()
try:
os.chdir(new_dir)
sys.path.insert(0, new_dir)
yield
finally:
del sys.path[0]
os.chdir(old_dir)
def setup_package():
root = os.path.abspath(os.path.dirname(__file__))
if len(sys.argv) > 1 and sys.argv[1] == 'clean':
return clean(root)
with chdir(root):
with open(os.path.join(root, 'thinc', 'about.py')) as f:
about = {}
exec(f.read(), about)
with io.open(os.path.join(root, 'README.rst'), encoding='utf8') as f:
readme = f.read()
include_dirs = [
get_python_inc(plat_specific=True),
os.path.join(root, 'include')]
if (ccompiler.new_compiler().compiler_type == 'msvc'
and msvccompiler.get_build_version() == 9):
include_dirs.append(os.path.join(root, 'include', 'msvc9'))
ext_modules = []
for mod_name in MOD_NAMES:
mod_path = mod_name.replace('.', '/') + '.cpp'
if mod_name.endswith('gpu_ops'):
continue
elif mod_name.endswith('openblas'):
ext = Openblas(mod_name, [mod_path], include_dirs=include_dirs)
else:
ext = Extension(mod_name, [mod_path],
language='c++', include_dirs=include_dirs)
ext_modules.append(ext)
if CUDA is None:
pass
else:
with chdir(root):
ext_modules.append(
Extension("thinc.neural.gpu_ops",
sources=["thinc/neural/gpu_ops.cpp", "include/_cuda_shim.cu"],
library_dirs=[CUDA['lib64']],
libraries=['cudart'],
language='c++',
runtime_library_dirs=[CUDA['lib64']],
# this syntax is specific to this build system
# we're only going to use certain compiler args with nvcc and not with gcc
# the implementation of this trick is in customize_compiler() below
extra_compile_args=['-arch=sm_30', '--ptxas-options=-v', '-c',
'--compiler-options', "'-fPIC'"],
include_dirs = include_dirs + [CUDA['include']]))
if not is_source_release(root):
generate_cython(root, 'thinc')
setup(
name=about['__title__'],
zip_safe=False,
packages=PACKAGES,
package_data={'': ['*.pyx', '*.pxd', '*.pxi', '*.cpp']},
description=about['__summary__'],
long_description=readme,
author=about['__author__'],
author_email=about['__email__'],
version=about['__version__'],
url=about['__uri__'],
license=about['__license__'],
ext_modules=ext_modules,
install_requires=[
'numpy>=1.7.0',
'msgpack>=0.5.6,<1.0.0',
'msgpack-numpy>=0.4.1,<1.0.0',
'murmurhash>=0.28.0,<0.29.0',
'cymem>=1.30.0,<1.32.0',
'preshed>=1.0.0,<2.0.0',
'cytoolz>=0.9.0,<0.10',
'wrapt>=1.10.0,<1.11.0',
'plac>=0.9.6,<1.0.0',
'tqdm>=4.10.0,<5.0.0',
'six>=1.10.0,<2.0.0',
'dill>=0.2.7,<0.3.0',
'pathlib==1.0.1; python_version < "3.4"'
],
extras_require={
'cuda': ['cupy>=4.0'],
'cuda80': ['cupy-cuda80>=4.0'],
'cuda90': ['cupy-cuda90>=4.0'],
'cuda91': ['cupy-cuda91>=4.0'],
},
classifiers=[
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: MIT License',
'Operating System :: POSIX :: Linux',
'Operating System :: MacOS :: MacOS X',
'Operating System :: Microsoft :: Windows',
'Programming Language :: Cython',
'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Topic :: Scientific/Engineering'],
cmdclass = {
'build_ext': build_ext_subclass},
)
# Detect CPU info. No modules in setup.py :(
#
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# Copyright (c) 2014-2018, Matthew Brennan Jones <[email protected]>
# Py-cpuinfo gets CPU info with pure Python 2 & 3
# It uses the MIT License
# It is hosted at: https://github.com/workhorsy/py-cpuinfo
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
CPUINFO_VERSION = (4, 0, 0)
import os, sys
import glob
import re
import time
import platform
import multiprocessing
import ctypes
import pickle
import base64
import subprocess
try:
import _winreg as winreg
except ImportError as err:
try:
import winreg
except ImportError as err:
pass
PY2 = sys.version_info[0] == 2
# Load hacks for Windows
if platform.system().lower() == 'windows':
# Monkey patch multiprocessing's Popen to fork properly on Windows Pyinstaller
# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Multiprocessing
try:
import multiprocessing.popen_spawn_win32 as forking
except ImportError as err:
try:
import multiprocessing.popen_fork as forking
except ImportError as err:
import multiprocessing.forking as forking
class _Popen(forking.Popen):
def __init__(self, *args, **kw):
if hasattr(sys, 'frozen'):
# We have to set original _MEIPASS2 value from sys._MEIPASS
# to get --onefile mode working.
os.putenv('_MEIPASS2', sys._MEIPASS)
try:
super(_Popen, self).__init__(*args, **kw)
finally:
if hasattr(sys, 'frozen'):
# On some platforms (e.g. AIX) 'os.unsetenv()' is not
# available. In those cases we cannot delete the variable
# but only set it to the empty string. The bootloader
# can handle this case.
if hasattr(os, 'unsetenv'):
os.unsetenv('_MEIPASS2')
else:
os.putenv('_MEIPASS2', '')
forking.Popen = _Popen
class DataSource(object):
bits = platform.architecture()[0]
cpu_count = multiprocessing.cpu_count()
is_windows = platform.system().lower() == 'windows'
raw_arch_string = platform.machine()
can_cpuid = True
@staticmethod
def has_proc_cpuinfo():
return os.path.exists('/proc/cpuinfo')
@staticmethod
def has_dmesg():
return len(program_paths('dmesg')) > 0
@staticmethod
def has_var_run_dmesg_boot():
uname = platform.system().strip().strip('"').strip("'").strip().lower()
return 'linux' in uname and os.path.exists('/var/run/dmesg.boot')
@staticmethod
def has_cpufreq_info():
return len(program_paths('cpufreq-info')) > 0
@staticmethod
def has_sestatus():
return len(program_paths('sestatus')) > 0
@staticmethod
def has_sysctl():
return len(program_paths('sysctl')) > 0
@staticmethod
def has_isainfo():
return len(program_paths('isainfo')) > 0
@staticmethod
def has_kstat():
return len(program_paths('kstat')) > 0
@staticmethod
def has_sysinfo():
return len(program_paths('sysinfo')) > 0
@staticmethod
def has_lscpu():
return len(program_paths('lscpu')) > 0
@staticmethod
def has_ibm_pa_features():
return len(program_paths('lsprop')) > 0
@staticmethod
def has_wmic():
returncode, output = run_and_get_stdout(['wmic', 'os', 'get', 'Version'])
return returncode == 0 and len(output) > 0
@staticmethod
def cat_proc_cpuinfo():
return run_and_get_stdout(['cat', '/proc/cpuinfo'])
@staticmethod
def cpufreq_info():
return run_and_get_stdout(['cpufreq-info'])
@staticmethod
def sestatus_allow_execheap():
return run_and_get_stdout(['sestatus', '-b'], ['grep', '-i', '"allow_execheap"'])[1].strip().lower().endswith('on')
@staticmethod
def sestatus_allow_execmem():
return run_and_get_stdout(['sestatus', '-b'], ['grep', '-i', '"allow_execmem"'])[1].strip().lower().endswith('on')
@staticmethod
def dmesg_a():
return run_and_get_stdout(['dmesg', '-a'])
@staticmethod
def cat_var_run_dmesg_boot():
return run_and_get_stdout(['cat', '/var/run/dmesg.boot'])
@staticmethod
def sysctl_machdep_cpu_hw_cpufrequency():
return run_and_get_stdout(['sysctl', 'machdep.cpu', 'hw.cpufrequency'])
@staticmethod
def isainfo_vb():
return run_and_get_stdout(['isainfo', '-vb'])
@staticmethod
def kstat_m_cpu_info():
return run_and_get_stdout(['kstat', '-m', 'cpu_info'])
@staticmethod
def sysinfo_cpu():
return run_and_get_stdout(['sysinfo', '-cpu'])
@staticmethod
def lscpu():
return run_and_get_stdout(['lscpu'])
@staticmethod
def ibm_pa_features():
ibm_features = glob.glob('/proc/device-tree/cpus/*/ibm,pa-features')
if ibm_features:
return run_and_get_stdout(['lsprop', ibm_features[0]])
@staticmethod
def wmic_cpu():
return run_and_get_stdout(['wmic', 'cpu', 'get', 'Name,CurrentClockSpeed,L2CacheSize,L3CacheSize,Description,Caption,Manufacturer', '/format:list'])
@staticmethod
def winreg_processor_brand():
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"Hardware\Description\System\CentralProcessor\0")
processor_brand = winreg.QueryValueEx(key, "ProcessorNameString")[0]
winreg.CloseKey(key)
return processor_brand
@staticmethod
def winreg_vendor_id():
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"Hardware\Description\System\CentralProcessor\0")
vendor_id = winreg.QueryValueEx(key, "VendorIdentifier")[0]
winreg.CloseKey(key)
return vendor_id
@staticmethod
def winreg_raw_arch_string():
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment")
raw_arch_string = winreg.QueryValueEx(key, "PROCESSOR_ARCHITECTURE")[0]
winreg.CloseKey(key)
return raw_arch_string
@staticmethod
def winreg_hz_actual():
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"Hardware\Description\System\CentralProcessor\0")
hz_actual = winreg.QueryValueEx(key, "~Mhz")[0]
winreg.CloseKey(key)
hz_actual = to_hz_string(hz_actual)
return hz_actual
@staticmethod
def winreg_feature_bits():
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"Hardware\Description\System\CentralProcessor\0")
feature_bits = winreg.QueryValueEx(key, "FeatureSet")[0]
winreg.CloseKey(key)
return feature_bits
def obj_to_b64(thing):
a = thing
b = pickle.dumps(a)
c = base64.b64encode(b)
d = c.decode('utf8')
return d
def b64_to_obj(thing):
try:
a = base64.b64decode(thing)
b = pickle.loads(a)
return b
except:
return {}
def run_and_get_stdout(command, pipe_command=None):
if not pipe_command:
p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
output = p1.communicate()[0]
if not PY2:
output = output.decode(encoding='UTF-8')
return p1.returncode, output
else:
p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
p2 = subprocess.Popen(pipe_command, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p1.stdout.close()
output = p2.communicate()[0]
if not PY2:
output = output.decode(encoding='UTF-8')
return p2.returncode, output
def program_paths(program_name):
paths = []
exts = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))
path = os.environ['PATH']
for p in os.environ['PATH'].split(os.pathsep):
p = os.path.join(p, program_name)
if os.access(p, os.X_OK):
paths.append(p)
for e in exts:
pext = p + e
if os.access(pext, os.X_OK):
paths.append(pext)
return paths
def _get_field_actual(cant_be_number, raw_string, field_names):
for line in raw_string.splitlines():
for field_name in field_names:
field_name = field_name.lower()
if ':' in line:
left, right = line.split(':', 1)
left = left.strip().lower()
right = right.strip()
if left == field_name and len(right) > 0:
if cant_be_number:
if not right.isdigit():
return right
else:
return right
return None
def _get_field(cant_be_number, raw_string, convert_to, default_value, *field_names):
retval = _get_field_actual(cant_be_number, raw_string, field_names)
# Convert the return value
if retval and convert_to:
try:
retval = convert_to(retval)
except:
retval = default_value
# Return the default if there is no return value
if retval is None:
retval = default_value
return retval
def _get_hz_string_from_brand(processor_brand):
# Just return 0 if the processor brand does not have the Hz
if not 'hz' in processor_brand.lower():
return (1, '0.0')
hz_brand = processor_brand.lower()
scale = 1
if hz_brand.endswith('mhz'):
scale = 6
elif hz_brand.endswith('ghz'):
scale = 9
if '@' in hz_brand:
hz_brand = hz_brand.split('@')[1]
else:
hz_brand = hz_brand.rsplit(None, 1)[1]
hz_brand = hz_brand.rstrip('mhz').rstrip('ghz').strip()
hz_brand = to_hz_string(hz_brand)
return (scale, hz_brand)
def to_friendly_hz(ticks, scale):
# Get the raw Hz as a string
left, right = to_raw_hz(ticks, scale)
ticks = '{0}.{1}'.format(left, right)
# Get the location of the dot, and remove said dot
dot_index = ticks.index('.')
ticks = ticks.replace('.', '')
# Get the Hz symbol and scale
symbol = "Hz"
scale = 0
if dot_index > 9:
symbol = "GHz"
scale = 9
elif dot_index > 6:
symbol = "MHz"
scale = 6
elif dot_index > 3:
symbol = "KHz"
scale = 3
# Get the Hz with the dot at the new scaled point
ticks = '{0}.{1}'.format(ticks[:-scale-1], ticks[-scale-1:])
# Format the ticks to have 4 numbers after the decimal
# and remove any superfluous zeroes.
ticks = '{0:.4f} {1}'.format(float(ticks), symbol)
ticks = ticks.rstrip('0')
return ticks
def to_raw_hz(ticks, scale):
# Scale the numbers
ticks = ticks.lstrip('0')
old_index = ticks.index('.')
ticks = ticks.replace('.', '')
ticks = ticks.ljust(scale + old_index+1, '0')
new_index = old_index + scale
ticks = '{0}.{1}'.format(ticks[:new_index], ticks[new_index:])
left, right = ticks.split('.')
left, right = int(left), int(right)
return (left, right)
def to_hz_string(ticks):
# Convert to string
ticks = '{0}'.format(ticks)
# Add decimal if missing
if '.' not in ticks:
ticks = '{0}.0'.format(ticks)
# Remove trailing zeros
ticks = ticks.rstrip('0')
# Add one trailing zero for empty right side
if ticks.endswith('.'):
ticks = '{0}0'.format(ticks)
return ticks
def to_friendly_bytes(input):
if not input:
return input
input = "{0}".format(input)
formats = {
r"^[0-9]+B$" : 'B',
r"^[0-9]+K$" : 'KB',
r"^[0-9]+M$" : 'MB',
r"^[0-9]+G$" : 'GB'
}
for pattern, friendly_size in formats.items():
if re.match(pattern, input):
return "{0} {1}".format(input[ : -1].strip(), friendly_size)
return input
def _parse_cpu_string(cpu_string):
# Get location of fields at end of string
fields_index = cpu_string.find('(', cpu_string.find('@'))
#print(fields_index)
# Processor Brand
processor_brand = cpu_string
if fields_index != -1:
processor_brand = cpu_string[0 : fields_index].strip()
#print('processor_brand: ', processor_brand)
fields = None
if fields_index != -1:
fields = cpu_string[fields_index : ]
#print('fields: ', fields)
# Hz
scale, hz_brand = _get_hz_string_from_brand(processor_brand)
# Various fields
vendor_id, stepping, model, family = (None, None, None, None)
if fields:
try:
fields = fields.rsplit('(', 1)[1].split(')')[0].split(',')
fields = [f.strip().lower() for f in fields]
fields = [f.split(':') for f in fields]
fields = [{f[0].strip() : f[1].strip()} for f in fields]
#print('fields: ', fields)
for field in fields:
name = list(field.keys())[0]
value = list(field.values())[0]
#print('name:{0}, value:{1}'.format(name, value))
if name == 'origin':
vendor_id = value.strip('"')
elif name == 'stepping':
stepping = int(value.lstrip('0x'), 16)
elif name == 'model':
model = int(value.lstrip('0x'), 16)
elif name in ['fam', 'family']:
family = int(value.lstrip('0x'), 16)
except:
#raise
pass
return (processor_brand, hz_brand, scale, vendor_id, stepping, model, family)
def _parse_dmesg_output(output):
try:
# Get all the dmesg lines that might contain a CPU string
lines = output.split(' CPU0:')[1:] + \
output.split(' CPU1:')[1:] + \
output.split(' CPU:')[1:] + \
output.split('\nCPU0:')[1:] + \
output.split('\nCPU1:')[1:] + \
output.split('\nCPU:')[1:]
lines = [l.split('\n')[0].strip() for l in lines]
# Convert the lines to CPU strings
cpu_strings = [_parse_cpu_string(l) for l in lines]