Skip to content

Commit

Permalink
Faster code generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@ctiller
ctiller committed Feb 25, 2016
1 parent 86b258c commit 560c901
Show file tree
Hide file tree
Showing 6 changed files with 160 additions and 133 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ coverage

# cache for run_tests.py
.run_tests_cache
.preprocessed_build

# emacs temp files
*~
Expand Down
10 changes: 5 additions & 5 deletions config.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,9 @@ if test "$PHP_GRPC" != "no"; then
src/core/support/env_linux.c \
src/core/support/env_posix.c \
src/core/support/env_win32.c \
src/core/support/file.c \
src/core/support/file_posix.c \
src/core/support/file_win32.c \
src/core/support/histogram.c \
src/core/support/host_port.c \
src/core/support/load_file.c \
src/core/support/log.c \
src/core/support/log_android.c \
src/core/support/log_linux.c \
Expand All @@ -78,6 +76,8 @@ if test "$PHP_GRPC" != "no"; then
src/core/support/time_precise.c \
src/core/support/time_win32.c \
src/core/support/tls_pthread.c \
src/core/support/tmpfile_posix.c \
src/core/support/tmpfile_win32.c \
src/core/support/wrap_memcpy.c \
src/core/census/grpc_context.c \
src/core/census/grpc_filter.c \
Expand Down Expand Up @@ -109,7 +109,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/client_config/subchannel_factory.c \
src/core/client_config/subchannel_index.c \
src/core/client_config/uri_parser.c \
src/core/compression/algorithm.c \
src/core/compression/compression_algorithm.c \
src/core/compression/message_compress.c \
src/core/debug/trace.c \
src/core/httpcli/format_request.c \
Expand Down Expand Up @@ -210,7 +210,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/transport/transport.c \
src/core/transport/transport_op_string.c \
src/core/httpcli/httpcli_security_connector.c \
src/core/security/base64.c \
src/core/security/b64.c \
src/core/security/client_auth_filter.c \
src/core/security/credentials.c \
src/core/security/credentials_metadata.c \
Expand Down
192 changes: 96 additions & 96 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,102 +421,6 @@
"third_party/zlib/trees.c",
"third_party/zlib/uncompr.c",
"third_party/zlib/zutil.c",
"include/grpc/support/alloc.h",
"include/grpc/support/atm.h",
"include/grpc/support/atm_gcc_atomic.h",
"include/grpc/support/atm_gcc_sync.h",
"include/grpc/support/atm_win32.h",
"include/grpc/support/avl.h",
"include/grpc/support/cmdline.h",
"include/grpc/support/cpu.h",
"include/grpc/support/histogram.h",
"include/grpc/support/host_port.h",
"include/grpc/support/log.h",
"include/grpc/support/log_win32.h",
"include/grpc/support/port_platform.h",
"include/grpc/support/slice.h",
"include/grpc/support/slice_buffer.h",
"include/grpc/support/string_util.h",
"include/grpc/support/subprocess.h",
"include/grpc/support/sync.h",
"include/grpc/support/sync_generic.h",
"include/grpc/support/sync_posix.h",
"include/grpc/support/sync_win32.h",
"include/grpc/support/thd.h",
"include/grpc/support/time.h",
"include/grpc/support/tls.h",
"include/grpc/support/tls_gcc.h",
"include/grpc/support/tls_msvc.h",
"include/grpc/support/tls_pthread.h",
"include/grpc/support/useful.h",
"include/grpc/impl/codegen/alloc.h",
"include/grpc/impl/codegen/atm.h",
"include/grpc/impl/codegen/atm_gcc_atomic.h",
"include/grpc/impl/codegen/atm_gcc_sync.h",
"include/grpc/impl/codegen/atm_win32.h",
"include/grpc/impl/codegen/log.h",
"include/grpc/impl/codegen/port_platform.h",
"include/grpc/impl/codegen/slice.h",
"include/grpc/impl/codegen/slice_buffer.h",
"include/grpc/impl/codegen/sync.h",
"include/grpc/impl/codegen/sync_generic.h",
"include/grpc/impl/codegen/sync_posix.h",
"include/grpc/impl/codegen/sync_win32.h",
"include/grpc/impl/codegen/time.h",
"src/core/profiling/timers.h",
"src/core/support/block_annotate.h",
"src/core/support/env.h",
"src/core/support/load_file.h",
"src/core/support/murmur_hash.h",
"src/core/support/stack_lockfree.h",
"src/core/support/string.h",
"src/core/support/string_win32.h",
"src/core/support/thd_internal.h",
"src/core/support/time_precise.h",
"src/core/support/tmpfile.h",
"src/core/profiling/basic_timers.c",
"src/core/profiling/stap_timers.c",
"src/core/support/alloc.c",
"src/core/support/avl.c",
"src/core/support/cmdline.c",
"src/core/support/cpu_iphone.c",
"src/core/support/cpu_linux.c",
"src/core/support/cpu_posix.c",
"src/core/support/cpu_windows.c",
"src/core/support/env_linux.c",
"src/core/support/env_posix.c",
"src/core/support/env_win32.c",
"src/core/support/histogram.c",
"src/core/support/host_port.c",
"src/core/support/load_file.c",
"src/core/support/log.c",
"src/core/support/log_android.c",
"src/core/support/log_linux.c",
"src/core/support/log_posix.c",
"src/core/support/log_win32.c",
"src/core/support/murmur_hash.c",
"src/core/support/slice.c",
"src/core/support/slice_buffer.c",
"src/core/support/stack_lockfree.c",
"src/core/support/string.c",
"src/core/support/string_posix.c",
"src/core/support/string_win32.c",
"src/core/support/subprocess_posix.c",
"src/core/support/subprocess_windows.c",
"src/core/support/sync.c",
"src/core/support/sync_posix.c",
"src/core/support/sync_win32.c",
"src/core/support/thd.c",
"src/core/support/thd_posix.c",
"src/core/support/thd_win32.c",
"src/core/support/time.c",
"src/core/support/time_posix.c",
"src/core/support/time_precise.c",
"src/core/support/time_win32.c",
"src/core/support/tls_pthread.c",
"src/core/support/tmpfile_posix.c",
"src/core/support/tmpfile_win32.c",
"src/core/support/wrap_memcpy.c",
"third_party/boringssl/crypto/aes/internal.h",
"third_party/boringssl/crypto/asn1/asn1_locl.h",
"third_party/boringssl/crypto/bio/internal.h",
Expand Down Expand Up @@ -918,6 +822,102 @@
"third_party/boringssl/ssl/t1_enc.c",
"third_party/boringssl/ssl/t1_lib.c",
"third_party/boringssl/ssl/tls_record.c",
"include/grpc/support/alloc.h",
"include/grpc/support/atm.h",
"include/grpc/support/atm_gcc_atomic.h",
"include/grpc/support/atm_gcc_sync.h",
"include/grpc/support/atm_win32.h",
"include/grpc/support/avl.h",
"include/grpc/support/cmdline.h",
"include/grpc/support/cpu.h",
"include/grpc/support/histogram.h",
"include/grpc/support/host_port.h",
"include/grpc/support/log.h",
"include/grpc/support/log_win32.h",
"include/grpc/support/port_platform.h",
"include/grpc/support/slice.h",
"include/grpc/support/slice_buffer.h",
"include/grpc/support/string_util.h",
"include/grpc/support/subprocess.h",
"include/grpc/support/sync.h",
"include/grpc/support/sync_generic.h",
"include/grpc/support/sync_posix.h",
"include/grpc/support/sync_win32.h",
"include/grpc/support/thd.h",
"include/grpc/support/time.h",
"include/grpc/support/tls.h",
"include/grpc/support/tls_gcc.h",
"include/grpc/support/tls_msvc.h",
"include/grpc/support/tls_pthread.h",
"include/grpc/support/useful.h",
"include/grpc/impl/codegen/alloc.h",
"include/grpc/impl/codegen/atm.h",
"include/grpc/impl/codegen/atm_gcc_atomic.h",
"include/grpc/impl/codegen/atm_gcc_sync.h",
"include/grpc/impl/codegen/atm_win32.h",
"include/grpc/impl/codegen/log.h",
"include/grpc/impl/codegen/port_platform.h",
"include/grpc/impl/codegen/slice.h",
"include/grpc/impl/codegen/slice_buffer.h",
"include/grpc/impl/codegen/sync.h",
"include/grpc/impl/codegen/sync_generic.h",
"include/grpc/impl/codegen/sync_posix.h",
"include/grpc/impl/codegen/sync_win32.h",
"include/grpc/impl/codegen/time.h",
"src/core/profiling/timers.h",
"src/core/support/block_annotate.h",
"src/core/support/env.h",
"src/core/support/load_file.h",
"src/core/support/murmur_hash.h",
"src/core/support/stack_lockfree.h",
"src/core/support/string.h",
"src/core/support/string_win32.h",
"src/core/support/thd_internal.h",
"src/core/support/time_precise.h",
"src/core/support/tmpfile.h",
"src/core/profiling/basic_timers.c",
"src/core/profiling/stap_timers.c",
"src/core/support/alloc.c",
"src/core/support/avl.c",
"src/core/support/cmdline.c",
"src/core/support/cpu_iphone.c",
"src/core/support/cpu_linux.c",
"src/core/support/cpu_posix.c",
"src/core/support/cpu_windows.c",
"src/core/support/env_linux.c",
"src/core/support/env_posix.c",
"src/core/support/env_win32.c",
"src/core/support/histogram.c",
"src/core/support/host_port.c",
"src/core/support/load_file.c",
"src/core/support/log.c",
"src/core/support/log_android.c",
"src/core/support/log_linux.c",
"src/core/support/log_posix.c",
"src/core/support/log_win32.c",
"src/core/support/murmur_hash.c",
"src/core/support/slice.c",
"src/core/support/slice_buffer.c",
"src/core/support/stack_lockfree.c",
"src/core/support/string.c",
"src/core/support/string_posix.c",
"src/core/support/string_win32.c",
"src/core/support/subprocess_posix.c",
"src/core/support/subprocess_windows.c",
"src/core/support/sync.c",
"src/core/support/sync_posix.c",
"src/core/support/sync_win32.c",
"src/core/support/thd.c",
"src/core/support/thd_posix.c",
"src/core/support/thd_win32.c",
"src/core/support/time.c",
"src/core/support/time_posix.c",
"src/core/support/time_precise.c",
"src/core/support/time_win32.c",
"src/core/support/tls_pthread.c",
"src/core/support/tmpfile_posix.c",
"src/core/support/tmpfile_win32.c",
"src/core/support/wrap_memcpy.c",
"binding.gyp"
],
"main": "src/node/index.js",
Expand Down
19 changes: 10 additions & 9 deletions package.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<email>[email protected]</email>
<active>yes</active>
</lead>
<date>2016-02-23</date>
<date>2016-02-24</date>
<time>16:06:07</time>
<version>
<release>0.8.0</release>
Expand Down Expand Up @@ -95,13 +95,14 @@
<file baseinstalldir="/" name="src/core/profiling/timers.h" role="src" />
<file baseinstalldir="/" name="src/core/support/block_annotate.h" role="src" />
<file baseinstalldir="/" name="src/core/support/env.h" role="src" />
<file baseinstalldir="/" name="src/core/support/file.h" role="src" />
<file baseinstalldir="/" name="src/core/support/load_file.h" role="src" />
<file baseinstalldir="/" name="src/core/support/murmur_hash.h" role="src" />
<file baseinstalldir="/" name="src/core/support/stack_lockfree.h" role="src" />
<file baseinstalldir="/" name="src/core/support/string.h" role="src" />
<file baseinstalldir="/" name="src/core/support/string_win32.h" role="src" />
<file baseinstalldir="/" name="src/core/support/thd_internal.h" role="src" />
<file baseinstalldir="/" name="src/core/support/time_precise.h" role="src" />
<file baseinstalldir="/" name="src/core/support/tmpfile.h" role="src" />
<file baseinstalldir="/" name="src/core/profiling/basic_timers.c" role="src" />
<file baseinstalldir="/" name="src/core/profiling/stap_timers.c" role="src" />
<file baseinstalldir="/" name="src/core/support/alloc.c" role="src" />
Expand All @@ -114,11 +115,9 @@
<file baseinstalldir="/" name="src/core/support/env_linux.c" role="src" />
<file baseinstalldir="/" name="src/core/support/env_posix.c" role="src" />
<file baseinstalldir="/" name="src/core/support/env_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/file.c" role="src" />
<file baseinstalldir="/" name="src/core/support/file_posix.c" role="src" />
<file baseinstalldir="/" name="src/core/support/file_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/histogram.c" role="src" />
<file baseinstalldir="/" name="src/core/support/host_port.c" role="src" />
<file baseinstalldir="/" name="src/core/support/load_file.c" role="src" />
<file baseinstalldir="/" name="src/core/support/log.c" role="src" />
<file baseinstalldir="/" name="src/core/support/log_android.c" role="src" />
<file baseinstalldir="/" name="src/core/support/log_linux.c" role="src" />
Expand All @@ -144,6 +143,8 @@
<file baseinstalldir="/" name="src/core/support/time_precise.c" role="src" />
<file baseinstalldir="/" name="src/core/support/time_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/tls_pthread.c" role="src" />
<file baseinstalldir="/" name="src/core/support/tmpfile_posix.c" role="src" />
<file baseinstalldir="/" name="src/core/support/tmpfile_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/wrap_memcpy.c" role="src" />
<file baseinstalldir="/" name="include/grpc/grpc_security.h" role="src" />
<file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer.h" role="src" />
Expand Down Expand Up @@ -274,7 +275,7 @@
<file baseinstalldir="/" name="src/core/transport/transport.h" role="src" />
<file baseinstalldir="/" name="src/core/transport/transport_impl.h" role="src" />
<file baseinstalldir="/" name="src/core/security/auth_filters.h" role="src" />
<file baseinstalldir="/" name="src/core/security/base64.h" role="src" />
<file baseinstalldir="/" name="src/core/security/b64.h" role="src" />
<file baseinstalldir="/" name="src/core/security/credentials.h" role="src" />
<file baseinstalldir="/" name="src/core/security/handshake.h" role="src" />
<file baseinstalldir="/" name="src/core/security/json_token.h" role="src" />
Expand Down Expand Up @@ -324,7 +325,7 @@
<file baseinstalldir="/" name="src/core/client_config/subchannel_factory.c" role="src" />
<file baseinstalldir="/" name="src/core/client_config/subchannel_index.c" role="src" />
<file baseinstalldir="/" name="src/core/client_config/uri_parser.c" role="src" />
<file baseinstalldir="/" name="src/core/compression/algorithm.c" role="src" />
<file baseinstalldir="/" name="src/core/compression/compression_algorithm.c" role="src" />
<file baseinstalldir="/" name="src/core/compression/message_compress.c" role="src" />
<file baseinstalldir="/" name="src/core/debug/trace.c" role="src" />
<file baseinstalldir="/" name="src/core/httpcli/format_request.c" role="src" />
Expand Down Expand Up @@ -425,7 +426,7 @@
<file baseinstalldir="/" name="src/core/transport/transport.c" role="src" />
<file baseinstalldir="/" name="src/core/transport/transport_op_string.c" role="src" />
<file baseinstalldir="/" name="src/core/httpcli/httpcli_security_connector.c" role="src" />
<file baseinstalldir="/" name="src/core/security/base64.c" role="src" />
<file baseinstalldir="/" name="src/core/security/b64.c" role="src" />
<file baseinstalldir="/" name="src/core/security/client_auth_filter.c" role="src" />
<file baseinstalldir="/" name="src/core/security/credentials.c" role="src" />
<file baseinstalldir="/" name="src/core/security/credentials_metadata.c" role="src" />
Expand Down Expand Up @@ -988,7 +989,7 @@ Update to wrap gRPC C Core version 0.10.0
<release>beta</release>
<api>beta</api>
</stability>
<date>2016-02-23</date>
<date>2016-02-24</date>
<license>BSD</license>
<notes>
- Simplify gRPC PHP installation #4517
Expand Down
31 changes: 21 additions & 10 deletions tools/buildgen/generate_projects.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@
os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..'))

argp = argparse.ArgumentParser()
argp.add_argument('json', nargs='+')
argp.add_argument('build_files', nargs='+', default=[])
argp.add_argument('--templates', nargs='+', default=[])
argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int)
args = argp.parse_args()

json = args.json
json = args.build_files

test = {} if 'TEST' in os.environ else None

Expand All @@ -62,21 +62,31 @@
for f in files:
templates.append(os.path.join(root, f))

pre_jobs = []
base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py']
cmd = base_cmd[:]
for plugin in plugins:
cmd.append('-p')
cmd.append(plugin)
for js in json:
cmd.append('-d')
cmd.append(js)
cmd.append('-w')
preprocessed_build = '.preprocessed_build'
cmd.append(preprocessed_build)
pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None))

jobs = []
for template in templates:
for template in reversed(sorted(templates)):
root, f = os.path.split(template)
if os.path.splitext(f)[1] == '.template':
out_dir = '.' + root[len('templates'):]
out = out_dir + '/' + os.path.splitext(f)[0]
if not os.path.exists(out_dir):
os.makedirs(out_dir)
cmd = ['python2.7', 'tools/buildgen/mako_renderer.py']
for plugin in plugins:
cmd.append('-p')
cmd.append(plugin)
for js in json:
cmd.append('-d')
cmd.append(js)
cmd = base_cmd[:]
cmd.append('-P')
cmd.append(preprocessed_build)
cmd.append('-o')
if test is None:
cmd.append(out)
Expand All @@ -88,6 +98,7 @@
cmd.append(root + '/' + f)
jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None))

jobset.run(pre_jobs, maxjobs=args.jobs)
jobset.run(jobs, maxjobs=args.jobs)

if test is not None:
Expand Down
Loading

0 comments on commit 560c901

Please sign in to comment.