curvefs: devops

.gitignore

@@ -79,3 +79,11 @@ runlog/
 
 # mac
 .DS_Store
+
+# curvefs
+curvefs/devops/projects/
+curvefs/devops/ssh/
+curvefs/devops/inventory
+*.pyc
+.facts/
+*retry

curvefs/Makefile

@@ -0,0 +1,91 @@
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+.PHONY: build install deploy core config topo start stop reload status clean mount umount
+
+prefix?= "$(PWD)/devops/projects"
+release?= 0
+only?= "*"
+hosts?= "*"
+
+define deploy_begin
+	@bash util/deploy.sh begin
+endef
+
+define deploy_end
+	@bash util/deploy.sh end
+endef
+
+define only_service
+	$(call deploy_begin)
+	@bash util/deploy.sh --hosts=$(hosts) --only="etcd" --tags=$(1)
+	@bash util/deploy.sh --hosts=$(hosts) --only="mds" --tags=$(1)
+	@bash util/deploy.sh --hosts=$(hosts) --only="metaserver" --tags=$(1)
+	$(call deploy_end)
+endef
+
+define only_specify
+	$(call deploy_begin)
+	@bash util/deploy.sh --hosts=$(hosts) --only=$(1) --tags=$(2)
+	$(call deploy_end)
+endef
+
+build:
+	@bash util/build.sh --only=$(only) --release=$(release)
+
+install:
+	@bash util/install.sh --prefix=$(prefix) --only=$(only)
+
+deploy:
+	$(call deploy_begin)
+	@bash util/deploy.sh --hosts="etcd" --only="etcd" --tags="core,config,start"
+	@bash util/deploy.sh --hosts="mds" --only="mds" --tags="core,config,start"
+	@bash util/deploy.sh --hosts="metaserver" --only="metaserver" --tags="core,config,start"
+	@bash util/deploy.sh --hosts="tools" --only="tools" --tags="core,config,topo"
+	@bash util/deploy.sh --hosts="client" --only="client" --tags="core,config"
+	$(call deploy_end)
+
+core:
+	$(call only_specify,$(only),core)
+
+config:
+	$(call only_specify,$(only),config)
+
+topo:
+	$(call only_specify,tools,topo)
+
+start:
+ifeq ($(only), "*")
+	$(call only_service,start)
+else
+	$(call only_specify,$(only),start)
+endif
+
+stop:
+ifeq ($(only), "*")
+	$(call only_service,stop)
+else
+	$(call only_specify,$(only),stop)
+endif
+
+reload:
+ifeq ($(only), "*")
+	$(call only_service,restart)
+else
+	$(call only_specify,$(only),restart)
+endif
+
+status:
+ifeq ($(only), "*")
+	$(call only_service,status)
+else
+	$(call only_specify,$(only),status)
+endif
+
+clean:
+	$(call only_specify,$(only),clean)
+
+mount:
+	$(call only_specify,client,mount)
+
+umount:
+	$(call only_specify,client,umount)

curvefs/README.md

@@ -0,0 +1,126 @@
+CURVEFS
+===
+
+Curve FileSystem
+
+Table of Contents
+===
+* [Requirement](#requirement)
+* [Quick Start](#quick-start)
+* [Devops](#devops)
+* [Hint](#hint)
+
+Requirement
+===
+
+* bazel
+* ansible
+* linux*
+
+[Back to Toc](#table-of-contents)
+
+Quick Start
+===
+
+step 1: clone repository, run build and install:
+
+```
+$ git clone https://github.com/opencurve/curve.git
+$ cd curve/curvefs
+$ make build
+$ make install
+$ make install only=etcd
+```
+
+step 2: edit ansible config file, inventory file and client config file:
+
+* devops/ansible.cfg:
+    * `remoter_user`: we use this user to log in to the remote machine, create file and start service
+    * `private_key_file`: paste `remote_user`'s private key to `devops/ssh/pub_rsa` (you can also save the private key anywhere locally and point the `private_key_file` to it)
+    * Please make sure that the mode of private key file which `private_key_file` specfied is `600` (`-rw-------`)
+    * Please make sure that the public key already exists in `remote_user`'s ssh file (`~/.ssh/authorized_keys`)
+* inventory/server.ini:
+    * You can specify which hosts to deploy which services
+    * Please specify the client mount path (`client_mount_path`), the path will create automatic if it not exist
+    * Please specify the filesystem name which client mount (`client_mount_fsname`), the filesystem will create automatic if it not exist
+    * The `tools` only used to create topology, you can select one of `mds` hosts
+* conf/{client.conf, metaserver.conf}
+    * `s3.ak`
+    * `s3.sk`
+    * `s3.endpoint`
+    * `s3.bucket_name`
+
+step 3: deploy all and mount curve filesystem:
+
+```
+$ make deploy
+$ make mount
+```
+
+once this is done, you can enter the mount path and do anything like local filesystem.
+
+[Back to Toc](#table-of-contents)
+
+Devops
+===
+
+| command                                  | description                                   |
+| :---                                     | :---                                          |
+| make build [only=ONLY] [release=1]       | compile                                       |
+| make install [only=ONLY] [prefix=PREFIX] | install                                       |
+| make deploy                              | deploy all                                    |
+| make core [only=ONLY] [hosts=HOSTS]      | sync binary file                              |
+| make config [only=ONLY] [hosts=HOSTS]    | sync config file                              |
+| make start [only=ONLY] [hosts=HOSTS]     | start service                                 |
+| make stop [only=ONLY] [hosts=HOSTS]      | stop service                                  |
+| make reload [only=ONLY] [hosts=HOSTS]    | restart service                               |
+| make status [only=ONLY] [hosts=HOSTS]    | show service status                           |
+| make clean [only=ONLY] [hosts=HOSTS]     | clean environment (include all created files) |
+| make topo                                | create topology                               |
+| make mount                               | mount curve filesystem                        |
+| make umount                              | umount curve filesystem                       |
+
+
+* If you want to execute action for specfied service, you can use `only` option, e.g: `make start only=mds`
+* If you want to execute action in specfied host, you can use `hosts` option, e.g: `make start hosts=machine1`
+* You can also specify both `only` and `hosts` option, e.g: `make start only=mds hosts=machine1:machine2`
+* The `only` option can be one of the following values: `etcd`、`mds`、`metaserver`、`space`、`client`、 `tools`
+
+[Back to Toc](#table-of-contents)
+
+HINT
+===
+
+* The default install prefix of projects is `devops/projects`
+* If you want to modify the service's config, you can modify the config under `conf` directory, then run `make config`
+* You can use `make status` to show service status, include active status, listen address, memory usage and etc:
+
+```
+$ make status only=mds
+
+curve-vm1:
+mds.service - CurveFS Mds
+    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
+  Main PID: 326373 (curvefs_mds)
+    Daemon: True
+    Listen: 0.0.0.0:26700, 0.0.0.0:27700
+       Mem: 51688 KB
+
+curve-vm2:
+mds.service - CurveFS Mds
+    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
+  Main PID: 326381 (curvefs_mds)
+    Daemon: True
+    Listen: 0.0.0.0:37700
+       Mem: 40688 KB
+
+curve-vm3:
+mds.service - CurveFS Mds
+    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
+  Main PID: 326401 (curvefs_mds)
+    Daemon: True
+    Listen: 0.0.0.0:17700
+       Mem: 55520 KB
+```
+
+[Back to Toc](#table-of-contents)

curvefs/conf/curvefs_client.conf → curvefs/conf/client.conf

@@ -6,7 +6,7 @@ mdsOpt.rpcRetryOpt.rpcRetryIntervalUS=50000
 mdsOpt.rpcRetryOpt.maxFailedTimesBeforeChangeAddr=5
 mdsOpt.rpcRetryOpt.normalRetryTimesBeforeTriggerWait=3
 mdsOpt.rpcRetryOpt.waitSleepMs=10000
-mdsOpt.rpcRetryOpt.addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702
+mdsOpt.rpcRetryOpt.addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702  # __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_port") }} __ANSIBLE_TEMPLATE__
 
 #metaCacheOpt
 metaCacheOpt.metacacheGetLeaderRetry=3
@@ -23,7 +23,7 @@ excutorOpt.minRetryTimesForceTimeoutBackoff=5
 excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
 
 #spaceserver
-spaceserver.spaceaddr=127.0.0.1:19999
+spaceserver.spaceaddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
 spaceserver.rpcTimeoutMs=1000
 
 #bdev
@@ -51,6 +51,10 @@ volume.fsBlockSize=4096
 #s3
 s3.blocksize=4194304
 s3.chunksize=67108864
+s3.intervalSec=3
+s3.flushInterval=5
+s3.writeCacheMaxByte=10737418240
+s3.lruCapacity=1000
 s3.endpoint=
 s3.bucket_name=
 s3.ak=

curvefs/conf/etcd.conf

@@ -0,0 +1,113 @@
+# This is the configuration file for the etcd server.
+
+# Human-readable name for this member.
+name:  # __ANSIBLE_TEMPLATE__ {{ inventory_hostname }} __ANSIBLE_TEMPLATE__
+
+# Path to the data directory.
+data-dir: # __ANSIBLE_TEMPLATE__ {{ curvefs_etcd_data_dir }} __ANSIBLE_TEMPLATE__
+
+# Path to the dedicated wal directory.
+wal-dir:  # __ANSIBLE_TEMPLATE__ {{ curvefs_etcd_wal_dir }} __ANSIBLE_TEMPLATE__
+
+# Number of committed transactions to trigger a snapshot to disk.
+snapshot-count: 10000
+
+# Time (in milliseconds) of a heartbeat interval.
+heartbeat-interval: 100
+
+# Time (in milliseconds) for an election to timeout.
+election-timeout: 1000
+quota-backend-bytes: 0
+
+# List of comma separated URLs to listen on for peer traffic.
+listen-peer-urls:  # __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_peer_port }} __ANSIBLE_TEMPLATE__
+
+# List of comma separated URLs to listen on for client traffic.
+listen-client-urls:  # __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_client_port }} __ANSIBLE_TEMPLATE__
+
+# Maximum number of snapshot files to retain (0 is unlimited).
+max-snapshots: 5
+
+# Maximum number of wal files to retain (0 is unlimited).
+max-wals: 5
+
+# Comma-separated white list of origins for CORS (cross-origin resource sharing).
+cors:
+
+# List of this member's peer URLs to advertise to the rest of the cluster.
+# The URLs needed to be a comma-separated list.
+initial-advertise-peer-urls:  # __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_peer_port }} __ANSIBLE_TEMPLATE__
+
+# List of this member's client URLs to advertise to the public.
+# The URLs needed to be a comma-separated list.
+advertise-client-urls:  # __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_client_port }} __ANSIBLE_TEMPLATE__
+
+# Discovery URL used to bootstrap the cluster.
+discovery:
+
+# Valid values include 'exit', 'proxy'
+discovery-fallback: proxy
+
+# HTTP proxy to use for traffic to discovery service.
+discovery-proxy:
+
+# DNS domain used to bootstrap initial cluster.
+discovery-srv:
+
+# Initial cluster configuration for bootstrapping.
+initial-cluster:  # __ANSIBLE_TEMPLATE__ {{ groups.etcd | join_etcd_peer(hostvars) }} __ANSIBLE_TEMPLATE__
+
+# Initial cluster token for the etcd cluster during bootstrap.
+initial-cluster-token: etcd-cluster
+
+# Initial cluster state ('new' or 'existing').
+initial-cluster-state: new
+
+# Reject reconfiguration requests that would cause quorum loss.
+strict-reconfig-check: False
+
+# Accept etcd V2 client requests
+enable-v2: True
+
+# Enable runtime profiling data via HTTP server
+enable-pprof: True
+
+# Valid values include 'on', 'readonly', 'off'
+proxy: 'off'
+
+# Time (in milliseconds) an endpoint will be held in a failed state.
+proxy-failure-wait: 5000
+
+# Time (in milliseconds) of the endpoints refresh interval.
+proxy-refresh-interval: 30000
+
+# Time (in milliseconds) for a dial to timeout.
+proxy-dial-timeout: 1000
+
+# Time (in milliseconds) for a write to timeout.
+proxy-write-timeout: 5000
+
+# Time (in milliseconds) for a read to timeout.
+proxy-read-timeout: 0
+
+# Enable debug-level logging for etcd.
+debug: False
+
+logger: zap
+
+# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
+log-outputs: [stderr]
+
+# Force to create a new one member cluster.
+force-new-cluster: False
+
+auto-compaction-mode: periodic
+auto-compaction-retention: "1"
+
+# Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
+metrics: extensive
+
+# Enable to run an additional Raft election phase.
+pre-vote: True
+
+enable-grpc-gateway: True