From 570bc5655ed7a97bafe0a135e95fdf2aa4967c84 Mon Sep 17 00:00:00 2001 From: wubenqi Date: Sat, 28 Jul 2018 18:43:39 +0800 Subject: [PATCH] Split nbfs to upload and document, fix upload big file bug. --- README.md | 40 +- .../crypto/md5_file.go | 34 +- baselib/grpc_util/rpc_client.go | 4 +- baselib/net2/tcp_client_group_manager.go | 2 +- biz/core/photo/model.go | 8 +- doc/build.md | 29 +- proto/mtproto/mtproto_http_proxy_codec.go | 4 +- scripts/build.sh | 12 +- server/access/auth_key/server/handshake.go | 9 +- .../session/server/client_session_handler.go | 11 +- .../session/server/client_session_manager.go | 6 +- server/access/session/session.toml | 2 +- .../rpc/account.getWallPapers_handler.go | 4 +- server/biz_server/biz_server.toml | 2 +- .../rpc/messages.editChatPhoto_handler.go | 4 +- .../rpc/messages.getStickerSet_handler.go | 4 +- .../rpc/messages.sendMedia_handler.go | 10 +- server/biz_server/messenger.go | 6 +- .../rpc/photos.getUserPhotos_handler.go | 4 +- .../rpc/photos.updateProfilePhoto_handler.go | 4 +- .../rpc/photos.uploadProfilePhoto_handler.go | 4 +- .../users/rpc/users.getFullUser_handler.go | 4 +- server/nbfs/README.md | 1 - server/nbfs/biz/README.md | 0 server/nbfs/biz/core/README.md | 0 server/nbfs/biz/core/core.go | 89 ----- .../nbfs/biz/core/document/document_file.go | 212 ----------- server/nbfs/biz/core/file/file_logic.go | 95 ----- server/nbfs/biz/core/file/file_part_logic.go | 229 ----------- .../biz/core/file/file_part_logic_test.go | 82 ---- server/nbfs/biz/core/photo/photo.go | 314 ---------------- server/nbfs/biz/core/photo/photo_file.go | 345 ----------------- server/nbfs/biz/core/photo/photo_test.go | 70 ---- .../biz/dal/dao/mysql_dao/file_parts_dao.go | 134 ------- .../nbfs/biz/dal/dao/mysql_dao/files_dao.go | 90 ----- server/nbfs/biz/dal/dataobject/files_do.go | 35 -- server/nbfs/biz/dal/tables/file_parts.xml | 31 -- server/nbfs/biz/dal/tables/files.xml | 22 -- server/nbfs/nbfs.go | 86 ----- server/nbfs/photo/rpc/photo_service_impl.go | 313 --------------- server/nbfs/upload/rpc/README.md | 7 - server/upload/README.md | 12 + server/upload/server/conf.go | 52 +++ .../upload.getCdnFileHashes_handler.go | 2 +- .../server}/upload.getCdnFile_handler.go | 2 +- .../server}/upload.getFile_handler.go | 40 +- .../server}/upload.getWebFile_handler.go | 2 +- .../server}/upload.reuploadCdnFile_handler.go | 2 +- .../server}/upload.saveBigFilePart_handler.go | 15 +- .../server}/upload.saveFilePart_handler.go | 15 +- server/upload/server/upload_server.go | 68 ++++ server/upload/server/upload_service_impl.go | 40 ++ .../{nbfs/biz/base/id.go => upload/upload.go} | 30 +- server/upload/upload.toml | 18 + service/document/README.md | 7 + .../document/biz/core/core.go | 10 +- .../biz/core/document/document_file.go | 111 ++++++ .../biz/core/document/document_file_test.go | 0 service/document/biz/core/document/model.go | 62 +++ service/document/biz/core/photo/model.go | 59 +++ service/document/biz/core/photo/photo_file.go | 206 ++++++++++ .../biz/core/photo/photo_file_test.go | 5 +- .../document}/biz/dal/README.md | 0 .../document}/biz/dal/dalgen/dalgen.sh | 0 .../document}/biz/dal/dalgen/dalgen.toml | 0 .../document}/biz/dal/dalgen/dalgen_all.sh | 0 .../biz/dal/dalgen/nebula-dal-generator | Bin .../document}/biz/dal/dalgen/tpl/gen_dao.tpl | 4 +- .../document}/biz/dal/dalgen/tpl/gen_do.tpl | 0 .../document}/biz/dal/dao/dao_manager.go | 0 .../biz/dal/dao/mysql_dao/documents_dao.go | 2 +- .../document}/biz/dal/dao/mysql_dao/gofmt.sh | 0 .../biz/dal/dao/mysql_dao/photo_datas_dao.go | 2 +- .../biz/dal/dataobject/documents_do.go | 2 + .../document}/biz/dal/dataobject/gofmt.sh | 0 .../biz/dal/dataobject/photo_datas_do.go | 2 + .../document}/biz/dal/tables/documents.xml | 0 .../document}/biz/dal/tables/photo_datas.xml | 0 service/document/client/document.go | 61 +++ .../document/client/rpc_document_client.go | 6 +- .../photo.go => service/document/document.go | 18 +- .../document/document.toml | 22 +- service/document/proto/build2.sh | 7 + .../document/proto/document.proto | 12 +- service/document/service/conf.go | 60 +++ service/document/service/document_server.go | 70 ++++ .../document/service/documtnt_service_impl.go | 244 ++++++++++++ service/nbfs/cachefs/cachefs.go | 191 ++++++++++ service/nbfs/cachefs/document_file.go | 77 ++++ service/nbfs/cachefs/photo_file.go | 270 +++++++++++++ .../nbfs/cachefs}/test001.jpeg | Bin .../nbfs/cachefs}/test002.jpeg | Bin service/nbfs/nbfs/local_nbfs_facade.go | 355 ++++++++++++++++++ service/nbfs/nbfs/nbfs.go | 61 +++ .../nbfs/proto/nbfs.proto.go | 43 ++- 95 files changed, 2272 insertions(+), 2357 deletions(-) rename server/nbfs/biz/core/file/file_location.go => baselib/crypto/md5_file.go (54%) delete mode 100644 server/nbfs/README.md delete mode 100644 server/nbfs/biz/README.md delete mode 100644 server/nbfs/biz/core/README.md delete mode 100644 server/nbfs/biz/core/core.go delete mode 100644 server/nbfs/biz/core/document/document_file.go delete mode 100644 server/nbfs/biz/core/file/file_logic.go delete mode 100644 server/nbfs/biz/core/file/file_part_logic.go delete mode 100644 server/nbfs/biz/core/file/file_part_logic_test.go delete mode 100644 server/nbfs/biz/core/photo/photo.go delete mode 100644 server/nbfs/biz/core/photo/photo_file.go delete mode 100644 server/nbfs/biz/core/photo/photo_test.go delete mode 100644 server/nbfs/biz/dal/dao/mysql_dao/file_parts_dao.go delete mode 100644 server/nbfs/biz/dal/dao/mysql_dao/files_dao.go delete mode 100644 server/nbfs/biz/dal/dataobject/files_do.go delete mode 100644 server/nbfs/biz/dal/tables/file_parts.xml delete mode 100644 server/nbfs/biz/dal/tables/files.xml delete mode 100644 server/nbfs/nbfs.go delete mode 100644 server/nbfs/photo/rpc/photo_service_impl.go delete mode 100644 server/nbfs/upload/rpc/README.md create mode 100644 server/upload/README.md create mode 100644 server/upload/server/conf.go rename server/{nbfs/upload/rpc => upload/server}/upload.getCdnFileHashes_handler.go (98%) rename server/{nbfs/upload/rpc => upload/server}/upload.getCdnFile_handler.go (98%) rename server/{nbfs/upload/rpc => upload/server}/upload.getFile_handler.go (52%) rename server/{nbfs/upload/rpc => upload/server}/upload.getWebFile_handler.go (98%) rename server/{nbfs/upload/rpc => upload/server}/upload.reuploadCdnFile_handler.go (98%) rename server/{nbfs/upload/rpc => upload/server}/upload.saveBigFilePart_handler.go (77%) rename server/{nbfs/upload/rpc => upload/server}/upload.saveFilePart_handler.go (78%) create mode 100644 server/upload/server/upload_server.go create mode 100644 server/upload/server/upload_service_impl.go rename server/{nbfs/biz/base/id.go => upload/upload.go} (66%) create mode 100644 server/upload/upload.toml create mode 100644 service/document/README.md rename server/nbfs/upload/rpc/upload_service_impl.go => service/document/biz/core/core.go (79%) create mode 100644 service/document/biz/core/document/document_file.go rename {server/nbfs => service/document}/biz/core/document/document_file_test.go (100%) create mode 100644 service/document/biz/core/document/model.go create mode 100644 service/document/biz/core/photo/model.go create mode 100644 service/document/biz/core/photo/photo_file.go rename {server/nbfs => service/document}/biz/core/photo/photo_file_test.go (91%) rename {server/nbfs => service/document}/biz/dal/README.md (100%) rename {server/nbfs => service/document}/biz/dal/dalgen/dalgen.sh (100%) rename {server/nbfs => service/document}/biz/dal/dalgen/dalgen.toml (100%) rename {server/nbfs => service/document}/biz/dal/dalgen/dalgen_all.sh (100%) rename {server/nbfs => service/document}/biz/dal/dalgen/nebula-dal-generator (100%) rename {server/nbfs => service/document}/biz/dal/dalgen/tpl/gen_dao.tpl (98%) rename {server/nbfs => service/document}/biz/dal/dalgen/tpl/gen_do.tpl (100%) rename {server/nbfs => service/document}/biz/dal/dao/dao_manager.go (100%) rename {server/nbfs => service/document}/biz/dal/dao/mysql_dao/documents_dao.go (98%) rename {server/nbfs => service/document}/biz/dal/dao/mysql_dao/gofmt.sh (100%) rename {server/nbfs => service/document}/biz/dal/dao/mysql_dao/photo_datas_dao.go (98%) rename {server/nbfs => service/document}/biz/dal/dataobject/documents_do.go (94%) rename {server/nbfs => service/document}/biz/dal/dataobject/gofmt.sh (100%) rename {server/nbfs => service/document}/biz/dal/dataobject/photo_datas_do.go (94%) rename {server/nbfs => service/document}/biz/dal/tables/documents.xml (100%) rename {server/nbfs => service/document}/biz/dal/tables/photo_datas.xml (100%) create mode 100644 service/document/client/document.go rename server/nbfs/nbfs_client/nbfs_rpc_client.go => service/document/client/rpc_document_client.go (97%) rename server/nbfs/photo/photo.go => service/document/document.go (60%) rename server/nbfs/nbfs.toml => service/document/document.toml (67%) create mode 100755 service/document/proto/build2.sh rename server/nbfs/upload/upload.go => service/document/proto/document.proto (68%) create mode 100644 service/document/service/conf.go create mode 100644 service/document/service/document_server.go create mode 100644 service/document/service/documtnt_service_impl.go create mode 100644 service/nbfs/cachefs/cachefs.go create mode 100644 service/nbfs/cachefs/document_file.go create mode 100644 service/nbfs/cachefs/photo_file.go rename {server/nbfs/biz/core/file => service/nbfs/cachefs}/test001.jpeg (100%) rename {server/nbfs/biz/core/file => service/nbfs/cachefs}/test002.jpeg (100%) create mode 100644 service/nbfs/nbfs/local_nbfs_facade.go create mode 100644 service/nbfs/nbfs/nbfs.go rename server/nbfs/biz/dal/dataobject/file_parts_do.go => service/nbfs/proto/nbfs.proto.go (53%) diff --git a/README.md b/README.md index 7c669c64..66dd1bf0 100644 --- a/README.md +++ b/README.md @@ -42,11 +42,6 @@ Go语言非官方开源telegram服务端，包括但不限于如下一些特色 go get go build - 编译session - cd $GOPATH/src/github.com/nebulaim/telegramd/server/access/session - go get - go build - 编译auth_key cd $GOPATH/src/github.com/nebulaim/telegramd/server/access/auth_key go get @@ -57,44 +52,49 @@ Go语言非官方开源telegram服务端，包括但不限于如下一些特色 go get go build - 编译nbfs - cd $GOPATH/src/github.com/nebulaim/telegramd/server/nbfs + 编译upload + cd $GOPATH/src/github.com/nebulaim/telegramd/server/upload go get go build + 编译document + cd $GOPATH/src/github.com/nebulaim/telegramd/service/document + go get + go build + 编译biz_server cd $GOPATH/src/github.com/nebulaim/telegramd/server/biz_server go get go build + + 编译session + cd $GOPATH/src/github.com/nebulaim/telegramd/server/access/session + go get + go build ``` - 运行 ``` - cd $GOPATH/src/github.com/nebulaim/telegramd/server/access/frontend - ./frontend - cd $GOPATH/src/github.com/nebulaim/telegramd/server/access/auth_key ./auth_key cd $GOPATH/src/github.com/nebulaim/telegramd/server/sync ./sync - cd $GOPATH/src/github.com/nebulaim/telegramd/server/nbfs - mkdir /opt/nbfs/0 - mkdir /opt/nbfs/s - mkdir /opt/nbfs/m - mkdir /opt/nbfs/x - mkdir /opt/nbfs/y - mkdir /opt/nbfs/a - mkdir /opt/nbfs/b - mkdir /opt/nbfs/c - ./nbfs + cd $GOPATH/src/github.com/nebulaim/telegramd/server/upload + ./upload + + cd $GOPATH/src/github.com/nebulaim/telegramd/service/document + ./document cd $GOPATH/src/github.com/nebulaim/telegramd/server/biz_server ./biz_server cd $GOPATH/src/github.com/nebulaim/telegramd/server/access/session ./session + + cd $GOPATH/src/github.com/nebulaim/telegramd/server/access/frontend + ./frontend ``` #### 更多文档 diff --git a/server/nbfs/biz/core/file/file_location.go b/baselib/crypto/md5_file.go similarity index 54% rename from server/nbfs/biz/core/file/file_location.go rename to baselib/crypto/md5_file.go index 8f5ea5ba..675948ef 100644 --- a/server/nbfs/biz/core/file/file_location.go +++ b/baselib/crypto/md5_file.go @@ -15,11 +15,33 @@ * limitations under the License. */ -package file +package crypto -// inputFileLocation#14637196 volume_id:long local_id:int secret:long = InputFileLocation; -// inputEncryptedFileLocation#f5235d55 id:long access_hash:long = InputFileLocation; -// inputDocumentFileLocation#430f0724 id:long access_hash:long version:int = InputFileLocation; +import ( + "github.com/golang/glog" + "crypto/md5" + "fmt" + "os" + "io" +) -// fileLocationUnavailable#7c596b46 volume_id:long local_id:int secret:long = FileLocation; -// fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation; +// TODO(@benqi): remove to baselib +func CalcMd5File(filename string) (string, error) { + // fileName := core.NBFS_DATA_PATH + m.data.FilePath + f, err := os.Open(filename) + if err != nil { + glog.Error(err) + return "", err + } + + defer f.Close() + + md5Hash := md5.New() + if _, err := io.Copy(md5Hash, f); err != nil { + // fmt.Println("Copy", err) + glog.Error("Copy - ", err) + return "", err + } + + return fmt.Sprintf("%x", md5Hash.Sum(nil)), nil +} diff --git a/baselib/grpc_util/rpc_client.go b/baselib/grpc_util/rpc_client.go index a6053d6c..27b3195f 100644 --- a/baselib/grpc_util/rpc_client.go +++ b/baselib/grpc_util/rpc_client.go @@ -110,7 +110,7 @@ func (c *RPCClient) Invoke(rpcMetaData *RpcMetadata, object mtproto.TLObject) (m // return nil, err } - glog.Infof("Invoke - method: {%s}, req: {%v}", t.Method, object) + glog.Infof("Invoke - method: {%s}", t.Method) r := t.NewReplyFunc() glog.Infof("Invoke - NewReplyFunc: {%v}, t: {%v}", r, reflect.TypeOf(r)) @@ -140,7 +140,7 @@ func (c *RPCClient) Invoke(rpcMetaData *RpcMetadata, object mtproto.TLObject) (m } return nil, mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_INTERNAL), "INTERNAL_SERVER_ERROR") } else { - glog.Infof("Invoke - Invoke reply: {%v}\n", r) + // glog.Infof("Invoke - Invoke reply: {%v}\n", r) reply, ok := r.(mtproto.TLObject) glog.Infof("Invoke %s time: %d", t.Method, (time.Now().Unix() - rpcMetaData.ReceiveTime)) diff --git a/baselib/net2/tcp_client_group_manager.go b/baselib/net2/tcp_client_group_manager.go index 5ea61809..80cdf595 100644 --- a/baselib/net2/tcp_client_group_manager.go +++ b/baselib/net2/tcp_client_group_manager.go @@ -148,7 +148,7 @@ func (cgm *TcpClientGroupManager) SendDataToAddress(name, address string, msg in cgm.clientMapLock.RUnlock() - glog.Info("tcp_client_group_manager sendDataToAddress: {name: %s, conn: %s, msg: {%v}}", name, c, msg) + glog.Infof("tcp_client_group_manager sendDataToAddress: {name: %s, conn: %s, msg: {%v}}", name, c, msg) return c.Send(msg) } diff --git a/biz/core/photo/model.go b/biz/core/photo/model.go index 73cdc1ed..1733cb70 100644 --- a/biz/core/photo/model.go +++ b/biz/core/photo/model.go @@ -20,7 +20,7 @@ package photo import ( "github.com/nebulaim/telegramd/biz/core" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" + "github.com/nebulaim/telegramd/service/document/client" "time" ) @@ -39,7 +39,7 @@ func (m *PhotoModel) GetUserProfilePhoto(photoId int64) (photo *mtproto.UserProf if photoId == 0 { photo = mtproto.NewTLUserProfilePhotoEmpty().To_UserProfilePhoto() } else { - sizes, _ := nbfs_client.GetPhotoSizeList(photoId) + sizes, _ := document_client.GetPhotoSizeList(photoId) photo = MakeUserProfilePhoto(photoId, sizes) } @@ -50,7 +50,7 @@ func (m *PhotoModel) GetChatPhoto(photoId int64) (photo *mtproto.ChatPhoto) { if photoId == 0 { photo = mtproto.NewTLChatPhotoEmpty().To_ChatPhoto() } else { - sizes, _ := nbfs_client.GetPhotoSizeList(photoId) + sizes, _ := document_client.GetPhotoSizeList(photoId) photo = MakeChatPhoto(sizes) } @@ -61,7 +61,7 @@ func (m *PhotoModel) GetPhoto(photoId int64) (photo *mtproto.Photo) { if photoId == 0 { photo = mtproto.NewTLPhotoEmpty().To_Photo() } else { - sizes, _ := nbfs_client.GetPhotoSizeList(photoId) + sizes, _ := document_client.GetPhotoSizeList(photoId) if len(sizes) == 0 { photo = mtproto.NewTLPhotoEmpty().To_Photo() } else { diff --git a/doc/build.md b/doc/build.md index eb5ae131..dd1df2f7 100644 --- a/doc/build.md +++ b/doc/build.md @@ -88,7 +88,8 @@ if password is empty ignore this section otherwise add password to the following ``` $ $GOPATH/src/github.com/nebulaim/telegramd/server/access/auth_key/auth_key.toml $ $GOPATH/src/github.com/nebulaim/telegramd/server/sync/sync.toml -$ $GOPATH/src/github.com/nebulaim/telegramd/server/nbfs/nbfs.toml +$ $GOPATH/src/github.com/nebulaim/telegramd/server/upload/upload.toml +$ $GOPATH/src/github.com/nebulaim/telegramd/service/ocument/document.toml $ $GOPATH/src/github.com/nebulaim/telegramd/server/biz_server/biz_server.toml ``` set ***my-secret-pw*** in mysql dsn as follow: @@ -134,9 +135,16 @@ $ go get $ go build ``` -### build nbfs +### build upload ``` -$ cd $GOPATH/src/github.com/nebulaim/telegramd/server/nbfs +$ cd $GOPATH/src/github.com/nebulaim/telegramd/server/upload +$ go get +$ go build +``` + +### build document +``` +$ cd $GOPATH/src/github.com/nebulaim/telegramd/service/document $ go get $ go build ``` @@ -178,16 +186,11 @@ $ ./auth_key $ cd $GOPATH/src/github.com/nebulaim/telegramd/server/sync $ ./sync -$ cd $GOPATH/src/github.com/nebulaim/telegramd/server/nbfs -$ mkdir /opt/nbfs/0 -$ mkdir /opt/nbfs/s -$ mkdir /opt/nbfs/m -$ mkdir /opt/nbfs/x -$ mkdir /opt/nbfs/y -$ mkdir /opt/nbfs/a -$ mkdir /opt/nbfs/b -$ mkdir /opt/nbfs/c -$ ./nbfs +$ cd $GOPATH/src/github.com/nebulaim/telegramd/server/upload +$ ./upload + +$ cd $GOPATH/src/github.com/nebulaim/telegramd/service/document +$ ./document $ cd $GOPATH/src/github.com/nebulaim/telegramd/server/biz_server $ ./biz_server diff --git a/proto/mtproto/mtproto_http_proxy_codec.go b/proto/mtproto/mtproto_http_proxy_codec.go index abd8e8f8..0a1c0b6b 100644 --- a/proto/mtproto/mtproto_http_proxy_codec.go +++ b/proto/mtproto/mtproto_http_proxy_codec.go @@ -99,14 +99,14 @@ func (c *MTProtoHttpProxyCodec) Send(msg interface{}) error { "Access-Control-Allow-Origin": {"*"}, "Access-Control-Max-Age": {"1728000"}, "Cache-control": {"no-store"}, - "Connection": {"close"}, + "Connection": {"keep-alive"}, "Content-type": {"application/octet-stream"}, "Pragma": {"no-cache"}, "Strict-Transport-Security": {"max-age=15768000"}, }, ContentLength: int64(len(b)), Body: ioutil.NopCloser(bytes.NewReader(b)), - Close: true, + Close: false, } err := rsp.Write(c.conn) diff --git a/scripts/build.sh b/scripts/build.sh index d44e9cd3..35ed8bc9 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -26,11 +26,17 @@ go get go build ./sync & -echo "build nbfs ..." -cd ${telegramd}/server/nbfs +echo "build upload ..." +cd ${telegramd}/server/upload go get go build -./nbfs & +./upload & + +echo "build document ..." +cd ${telegramd}/server/document +go get +go build +./document & echo "build biz_server ..." cd ${telegramd}/server/biz_server diff --git a/server/access/auth_key/server/handshake.go b/server/access/auth_key/server/handshake.go index f6415c68..995af240 100644 --- a/server/access/auth_key/server/handshake.go +++ b/server/access/auth_key/server/handshake.go @@ -35,6 +35,7 @@ import ( "github.com/nebulaim/telegramd/server/access/auth_key/dal/dataobject" "math/big" "time" + "github.com/nebulaim/telegramd/baselib/bytes2" ) const ( @@ -251,14 +252,18 @@ func (s *handshake) onReq_DHParams(state *zproto.HandshakeState, request *mtprot // 客户端传输数据解析 // check Nonce if !bytes.Equal(request.Nonce, authKeyMD.Nonce) { - err = fmt.Errorf("onReq_DHParams - Invalid Nonce") + err = fmt.Errorf("onReq_DHParams - Invalid Nonce, req: %s, back: %s", + bytes2.HexDump(request.Nonce), + bytes2.HexDump(authKeyMD.Nonce)) glog.Error(err) return nil, err } // check ServerNonce if !bytes.Equal(request.ServerNonce, authKeyMD.ServerNonce) { - err = fmt.Errorf("onReq_DHParams - Wrong ServerNonce") + err = fmt.Errorf("onReq_DHParams - Wrong ServerNonce, req: %s, back: %s", + bytes2.HexDump(request.ServerNonce), + bytes2.HexDump(authKeyMD.ServerNonce)) glog.Error(err) return nil, err } diff --git a/server/access/session/server/client_session_handler.go b/server/access/session/server/client_session_handler.go index e2c9eda8..32ecdba5 100644 --- a/server/access/session/server/client_session_handler.go +++ b/server/access/session/server/client_session_handler.go @@ -26,6 +26,7 @@ import ( "github.com/nebulaim/telegramd/proto/zproto" "math/rand" "time" + "reflect" ) const ( @@ -457,7 +458,7 @@ func (c *clientSessionHandler) onMessageData(connID ClientConnID, md *zproto.ZPr ) for _, message := range messages { - glog.Info("onMessageData - ", message) + // glog.Info("onMessageData - ", message) if message.Object == nil { continue @@ -1055,25 +1056,25 @@ func (c *clientSessionHandler) onInvokeAfterMsgsExt(connID ClientConnID, md *zpr } func (c *clientSessionHandler) onInvokeWithoutUpdatesExt(connID ClientConnID, md *zproto.ZProtoMetadata, msgId int64, seqNo int32, request *TLInvokeWithoutUpdatesExt) bool { - glog.Infof("onInvokeWithoutUpdatesExt - request data: {sess: %s, conn_id: %s, md: %s, msg_id: %d, seq_no: %d, request: {%v}}", + glog.Infof("onInvokeWithoutUpdatesExt - request data: {sess: %s, conn_id: %s, md: %s, msg_id: %d, seq_no: %d, request: {%s}}", c, connID, md, msgId, seqNo, - request) + reflect.TypeOf(request)) return c.onRpcRequest(connID, md, msgId, seqNo, request.Query) } func (c *clientSessionHandler) onRpcRequest(connID ClientConnID, md *zproto.ZProtoMetadata, msgId int64, seqNo int32, object mtproto.TLObject) bool { - glog.Infof("onInvokeWithoutUpdatesExt - request data: {sess: %s, conn_id: %s, md: %s, msg_id: %d, seq_no: %d, request: {%v}}", + glog.Infof("onRpcRequest - request data: {sess: %s, conn_id: %s, md: %s, msg_id: %d, seq_no: %d, request: {%s}}", c, connID, md, msgId, seqNo, - object) + reflect.TypeOf(object)) // TODO(@benqi): sync AuthUserId?? requestMessage := &mtproto.TLMessage2{ diff --git a/server/access/session/server/client_session_manager.go b/server/access/session/server/client_session_manager.go index af38e9bb..58cda797 100644 --- a/server/access/session/server/client_session_manager.go +++ b/server/access/session/server/client_session_manager.go @@ -467,7 +467,7 @@ func (s *clientSessionManager) onRpcRequest(requests *rpcApiMessages) { // err.(*mtproto.TLRpcError) } else { - glog.Infof("OnMessage - rpc_result: {%v}\n", rpcResult) + // glog.Infof("OnMessage - rpc_result: {%v}\n", rpcResult) reply.Result = rpcResult } @@ -546,7 +546,7 @@ func extractClientMessage(msgId int64, seqNo int32, object mtproto.TLObject, mes // for _, m := range msgContainer.Messages { - glog.Info("processMsgContainer - request data: ", m) + // glog.Info("processMsgContainer - request data: ", m) if m.Object == nil { continue } @@ -639,7 +639,7 @@ func extractClientMessage(msgId int64, seqNo int32, object mtproto.TLObject, mes messages.messages = append(messages.messages, &mtproto.TLMessage2{MsgId: msgId, Seqno: seqNo, Object: invokeWithoutUpdatesExt}) default: - glog.Info("processOthers - request data: ", object) + // glog.Info("processOthers - request data: ", object) messages.messages = append(messages.messages, &mtproto.TLMessage2{MsgId: msgId, Seqno: seqNo, Object: object}) } } diff --git a/server/access/session/session.toml b/server/access/session/session.toml index 8d8e358d..e87466b9 100644 --- a/server/access/session/session.toml +++ b/server/access/session/session.toml @@ -28,7 +28,7 @@ etcdAddrs = ["http://127.0.0.1:2379"] balancer = "round_robin" [nbfsRpcClient] -serviceName = "nbfs" +serviceName = "upload" etcdAddrs = ["http://127.0.0.1:2379"] balancer = "round_robin" diff --git a/server/biz_server/account/rpc/account.getWallPapers_handler.go b/server/biz_server/account/rpc/account.getWallPapers_handler.go index a8d07e69..51a7fbd6 100644 --- a/server/biz_server/account/rpc/account.getWallPapers_handler.go +++ b/server/biz_server/account/rpc/account.getWallPapers_handler.go @@ -22,7 +22,7 @@ import ( "github.com/nebulaim/telegramd/baselib/grpc_util" "github.com/nebulaim/telegramd/baselib/logger" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" + "github.com/nebulaim/telegramd/service/document/client" "golang.org/x/net/context" ) @@ -44,7 +44,7 @@ func (s *AccountServiceImpl) AccountGetWallPapers(ctx context.Context, request * for _, wallData := range wallDataList { if wallData.Type == 0 { - szList, _ := nbfs_client.GetPhotoSizeList(wallData.PhotoId) + szList, _ := document_client.GetPhotoSizeList(wallData.PhotoId) wall := &mtproto.TLWallPaper{Data2: &mtproto.WallPaper_Data{ Id: wallData.Id, Title: wallData.Title, diff --git a/server/biz_server/biz_server.toml b/server/biz_server/biz_server.toml index 80236d57..a7fafdf8 100644 --- a/server/biz_server/biz_server.toml +++ b/server/biz_server/biz_server.toml @@ -22,7 +22,7 @@ tTL = "10s" #addr = "localhost:10002" [nbfsRpcClient] -serviceName = "nbfs" +serviceName = "document" etcdAddrs = ["http://127.0.0.1:2379"] balancer = "round_robin" diff --git a/server/biz_server/messages/rpc/messages.editChatPhoto_handler.go b/server/biz_server/messages/rpc/messages.editChatPhoto_handler.go index f5b4211e..061409ef 100644 --- a/server/biz_server/messages/rpc/messages.editChatPhoto_handler.go +++ b/server/biz_server/messages/rpc/messages.editChatPhoto_handler.go @@ -25,7 +25,7 @@ import ( "github.com/nebulaim/telegramd/biz/core" update2 "github.com/nebulaim/telegramd/biz/core/update" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" + "github.com/nebulaim/telegramd/service/document/client" "github.com/nebulaim/telegramd/server/sync/sync_client" "golang.org/x/net/context" "time" @@ -75,7 +75,7 @@ func (s *MessagesServiceImpl) MessagesEditChatPhoto(ctx context.Context, request case mtproto.TLConstructor_CRC32_inputChatUploadedPhoto: file := chatPhoto.GetData2().GetFile() // photoId = helper.NextSnowflakeId() - result, err := nbfs_client.UploadPhotoFile(md.AuthId, file) // photoId, file.GetData2().GetId(), file.GetData2().GetParts(), file.GetData2().GetName(), file.GetData2().GetMd5Checksum()) + result, err := document_client.UploadPhotoFile(md.AuthId, file) // photoId, file.GetData2().GetId(), file.GetData2().GetParts(), file.GetData2().GetName(), file.GetData2().GetMd5Checksum()) if err != nil { glog.Errorf("UploadPhoto error: %v", err) return nil, err diff --git a/server/biz_server/messages/rpc/messages.getStickerSet_handler.go b/server/biz_server/messages/rpc/messages.getStickerSet_handler.go index 03ee6c52..298880f1 100644 --- a/server/biz_server/messages/rpc/messages.getStickerSet_handler.go +++ b/server/biz_server/messages/rpc/messages.getStickerSet_handler.go @@ -23,7 +23,7 @@ import ( "github.com/nebulaim/telegramd/baselib/logger" "github.com/nebulaim/telegramd/biz/core/sticker" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" + "github.com/nebulaim/telegramd/service/document/client" "golang.org/x/net/context" ) @@ -43,7 +43,7 @@ func (s *MessagesServiceImpl) MessagesGetStickerSet(ctx context.Context, request if len(idList) == 0 { documents = []*mtproto.Document{} } else { - documents, err = nbfs_client.GetDocumentByIdList(idList) + documents, err = document_client.GetDocumentByIdList(idList) if err != nil { glog.Error(err) documents = []*mtproto.Document{} diff --git a/server/biz_server/messages/rpc/messages.sendMedia_handler.go b/server/biz_server/messages/rpc/messages.sendMedia_handler.go index 1dcdc50a..b5ebaac6 100644 --- a/server/biz_server/messages/rpc/messages.sendMedia_handler.go +++ b/server/biz_server/messages/rpc/messages.sendMedia_handler.go @@ -26,10 +26,10 @@ import ( message2 "github.com/nebulaim/telegramd/biz/core/message" "github.com/nebulaim/telegramd/biz/core/update" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" "github.com/nebulaim/telegramd/server/sync/sync_client" "golang.org/x/net/context" "time" + "github.com/nebulaim/telegramd/service/document/client" ) func makeGeoPointByInput(geoPoint *mtproto.InputGeoPoint) *mtproto.GeoPoint { @@ -57,7 +57,7 @@ func (s *MessagesServiceImpl) makeMediaByInputMedia(authKeyId int64, media *mtpr uploadedPhoto := media.To_InputMediaUploadedPhoto() file := uploadedPhoto.GetFile() - result, err := nbfs_client.UploadPhotoFile(authKeyId, file) + result, err := document_client.UploadPhotoFile(authKeyId, file) // , file.GetData2().GetId(), file.GetData2().GetParts(), file.GetData2().GetName(), file.GetData2().GetMd5Checksum()) if err != nil { glog.Errorf("UploadPhoto error: %v, by %s", err, logger.JsonDebugData(media)) @@ -84,7 +84,7 @@ func (s *MessagesServiceImpl) makeMediaByInputMedia(authKeyId int64, media *mtpr // inputPhoto#fb95c6c4 id:long access_hash:long = InputPhoto; //inputMediaPhoto#81fa373a flags:# id:InputPhoto caption:string ttl_seconds:flags.0?int = InputMedia; mediaPhoto := media.To_InputMediaPhoto() - sizeList, _ := nbfs_client.GetPhotoSizeList(mediaPhoto.GetId().GetData2().GetId()) + sizeList, _ := document_client.GetPhotoSizeList(mediaPhoto.GetId().GetData2().GetId()) photo := &mtproto.TLPhoto{Data2: &mtproto.Photo_Data{ Id: mediaPhoto.GetId().GetData2().GetId(), @@ -131,7 +131,7 @@ func (s *MessagesServiceImpl) makeMediaByInputMedia(authKeyId int64, media *mtpr case mtproto.TLConstructor_CRC32_inputMediaUploadedDocument: // inputMediaUploadedDocument#e39621fd flags:# file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; uploadedDocument := media.To_InputMediaUploadedDocument() - messageMedia, _ := nbfs_client.UploadedDocumentMedia(authKeyId, uploadedDocument) + messageMedia, _ := document_client.UploadedDocumentMedia(authKeyId, uploadedDocument) return messageMedia.To_MessageMedia() // id:InputDocument caption:string ttl_seconds:flags.0?int @@ -139,7 +139,7 @@ func (s *MessagesServiceImpl) makeMediaByInputMedia(authKeyId int64, media *mtpr // inputMediaDocument#5acb668e flags:# id:InputDocument caption:string ttl_seconds:flags.0?int = InputMedia; // document := media.To_InputMediaDocument() id := media.To_InputMediaDocument().GetId() - document3, _ := nbfs_client.GetDocumentById(id.GetData2().GetId(), id.GetData2().GetAccessHash()) + document3, _ := document_client.GetDocumentById(id.GetData2().GetId(), id.GetData2().GetAccessHash()) // messageMediaDocument#7c4414d3 flags:# document:flags.0?Document caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; messageMedia := &mtproto.TLMessageMediaDocument{Data2: &mtproto.MessageMedia_Data{ diff --git a/server/biz_server/messenger.go b/server/biz_server/messenger.go index 08a31e8d..0630b245 100644 --- a/server/biz_server/messenger.go +++ b/server/biz_server/messenger.go @@ -45,9 +45,9 @@ import ( stickers "github.com/nebulaim/telegramd/server/biz_server/stickers/rpc" updates "github.com/nebulaim/telegramd/server/biz_server/updates/rpc" users "github.com/nebulaim/telegramd/server/biz_server/users/rpc" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" "github.com/nebulaim/telegramd/server/sync/sync_client" "google.golang.org/grpc" + "github.com/nebulaim/telegramd/service/document/client" ) //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -112,7 +112,7 @@ func (s *messengerServer) Initialize() error { dao.InstallMysqlDAOManager(mysql_client.GetMysqlClientManager()) dao.InstallRedisDAOManager(redis_client.GetRedisClientManager()) - nbfs_client.InstallNbfsClient(Conf.NbfsRpcClient) + document_client.InstallNbfsClient(Conf.NbfsRpcClient) sync_client.InstallSyncClient(Conf.SyncRpcClient2) }) @@ -146,7 +146,7 @@ func (s *messengerServer) RunLoop() { func (s *messengerServer) Destroy() { glog.Infof("messengerServer - destroy...") //s.server.Stop() - //s.rpcServer.Stop() + s.rpcServer.Stop() //time.Sleep(1*time.Second) } diff --git a/server/biz_server/photos/rpc/photos.getUserPhotos_handler.go b/server/biz_server/photos/rpc/photos.getUserPhotos_handler.go index 74c7bb02..8b77c307 100644 --- a/server/biz_server/photos/rpc/photos.getUserPhotos_handler.go +++ b/server/biz_server/photos/rpc/photos.getUserPhotos_handler.go @@ -22,7 +22,7 @@ import ( "github.com/nebulaim/telegramd/baselib/grpc_util" "github.com/nebulaim/telegramd/baselib/logger" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" + "github.com/nebulaim/telegramd/service/document/client" "golang.org/x/net/context" "time" ) @@ -67,7 +67,7 @@ func (s *PhotosServiceImpl) PhotosGetUserPhotos(ctx context.Context, request *mt photoIdList := s.UserModel.GetUserPhotoIDList(userId) // idList := []int32{} for _, photoId := range photoIdList { - sizes, _ := nbfs_client.GetPhotoSizeList(photoId) + sizes, _ := document_client.GetPhotoSizeList(photoId) // photo2 := photo2.MakeUserProfilePhoto(photoId, sizes) photo := &mtproto.TLPhoto{Data2: &mtproto.Photo_Data{ Id: photoId, diff --git a/server/biz_server/photos/rpc/photos.updateProfilePhoto_handler.go b/server/biz_server/photos/rpc/photos.updateProfilePhoto_handler.go index 6eb4fd79..cee6c8a1 100644 --- a/server/biz_server/photos/rpc/photos.updateProfilePhoto_handler.go +++ b/server/biz_server/photos/rpc/photos.updateProfilePhoto_handler.go @@ -23,7 +23,7 @@ import ( "github.com/nebulaim/telegramd/baselib/logger" photo2 "github.com/nebulaim/telegramd/biz/core/photo" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" + "github.com/nebulaim/telegramd/service/document/client" "golang.org/x/net/context" ) @@ -42,7 +42,7 @@ func (s *PhotosServiceImpl) PhotosUpdateProfilePhoto(ctx context.Context, reques id := request.GetId().To_InputPhoto() // TODO(@benqi): check inputPhoto.access_hash - sizes, _ := nbfs_client.GetPhotoSizeList(id.GetId()) + sizes, _ := document_client.GetPhotoSizeList(id.GetId()) photo = photo2.MakeUserProfilePhoto(id.GetId(), sizes) } diff --git a/server/biz_server/photos/rpc/photos.uploadProfilePhoto_handler.go b/server/biz_server/photos/rpc/photos.uploadProfilePhoto_handler.go index d5923e35..500b9a56 100644 --- a/server/biz_server/photos/rpc/photos.uploadProfilePhoto_handler.go +++ b/server/biz_server/photos/rpc/photos.uploadProfilePhoto_handler.go @@ -23,7 +23,7 @@ import ( "github.com/nebulaim/telegramd/baselib/logger" photo2 "github.com/nebulaim/telegramd/biz/core/photo" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" + "github.com/nebulaim/telegramd/service/document/client" "github.com/nebulaim/telegramd/server/sync/sync_client" "golang.org/x/net/context" "time" @@ -41,7 +41,7 @@ func (s *PhotosServiceImpl) PhotosUploadProfilePhoto(ctx context.Context, reques file := request.GetFile() // uuid := helper.NextSnowflakeId() - result, err := nbfs_client.UploadPhotoFile(md.AuthId, file) // uuid, file.GetData2().GetId(), file.GetData2().GetParts(), file.GetData2().GetName(), file.GetData2().GetMd5Checksum()) + result, err := document_client.UploadPhotoFile(md.AuthId, file) // uuid, file.GetData2().GetId(), file.GetData2().GetParts(), file.GetData2().GetName(), file.GetData2().GetMd5Checksum()) if err != nil { glog.Errorf("UploadPhoto error: %v", err) return nil, err diff --git a/server/biz_server/users/rpc/users.getFullUser_handler.go b/server/biz_server/users/rpc/users.getFullUser_handler.go index e8618f6e..831a27cb 100644 --- a/server/biz_server/users/rpc/users.getFullUser_handler.go +++ b/server/biz_server/users/rpc/users.getFullUser_handler.go @@ -22,9 +22,9 @@ import ( "github.com/nebulaim/telegramd/baselib/grpc_util" "github.com/nebulaim/telegramd/baselib/logger" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/nbfs_client" "golang.org/x/net/context" "time" + "github.com/nebulaim/telegramd/service/document/client" ) // users.getFullUser#ca30a5b1 id:InputUser = UserFull; @@ -110,7 +110,7 @@ func (s *UsersServiceImpl) UsersGetFullUser(ctx context.Context, request *mtprot // profilePhoto := user.GetData2().GetPhoto() // profilePhoto.GetData2(). // photoId := user2.GetDefaultUserPhotoID(request.GetId().GetData2().GetUserId()) - sizes, _ := nbfs_client.GetPhotoSizeList(photoId) + sizes, _ := document_client.GetPhotoSizeList(photoId) // photo2 := photo2.MakeUserProfilePhoto(photoId, sizes) photo := &mtproto.TLPhoto{Data2: &mtproto.Photo_Data{ Id: photoId, diff --git a/server/nbfs/README.md b/server/nbfs/README.md deleted file mode 100644 index ccb72b1b..00000000 --- a/server/nbfs/README.md +++ /dev/null @@ -1 +0,0 @@ -# nebula简单文件系统 diff --git a/server/nbfs/biz/README.md b/server/nbfs/biz/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/server/nbfs/biz/core/README.md b/server/nbfs/biz/core/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/server/nbfs/biz/core/core.go b/server/nbfs/biz/core/core.go deleted file mode 100644 index a9486c6c..00000000 --- a/server/nbfs/biz/core/core.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2017, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package core - -import ( - "crypto/md5" - "fmt" - "github.com/golang/glog" - "github.com/nebulaim/telegramd/proto/mtproto" - "io" - "os" -) - -//const ( -// NBFS_DATA_PATH = "/opt/nbfs" -// // kMaxFilePartSize = 32768 -//) - -var NBFS_DATA_PATH = "/opt/nbfs" - -func InitNbfsDataPath(path string) { - NBFS_DATA_PATH = path -} - -func MakeStorageFileType(extName string) *mtproto.Storage_FileType { - fileType := &mtproto.Storage_FileType{Data2: &mtproto.Storage_FileType_Data{}} - - switch extName { - case ".partial": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_filePartial - case ".jpeg", ".jpg": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileJpeg - case ".gif": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileGif - case ".png": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_filePng - case ".pdf": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_filePdf - case ".mp3": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileMp3 - case ".mov": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileMov - case ".mp4": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileMp4 - case ".webp": - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileWebp - default: - // fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileUnknown - fileType.Constructor = mtproto.TLConstructor_CRC32_storage_filePartial - } - - return fileType -} - -// TODO(@benqi): remove to baselib -func CalcMd5File(filename string) (string, error) { - // fileName := core.NBFS_DATA_PATH + m.data.FilePath - f, err := os.Open(filename) - if err != nil { - glog.Error(err) - return "", err - } - - defer f.Close() - - md5Hash := md5.New() - if _, err := io.Copy(md5Hash, f); err != nil { - // fmt.Println("Copy", err) - glog.Error("Copy - ", err) - return "", err - } - - return fmt.Sprintf("%x", md5Hash.Sum(nil)), nil -} diff --git a/server/nbfs/biz/core/document/document_file.go b/server/nbfs/biz/core/document/document_file.go deleted file mode 100644 index 4eb441af..00000000 --- a/server/nbfs/biz/core/document/document_file.go +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package document - -import ( - "encoding/json" - "fmt" - "github.com/golang/glog" - "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/core" - "github.com/nebulaim/telegramd/server/nbfs/biz/core/photo" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dao" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" - "math/rand" - "os" - "time" -) - -type documentData struct { - *dataobject.DocumentsDO -} - -func DoUploadedDocumentFile(documentId int64, filePath string, fileSize int32, uploadedFileName, extName, mimeType string, thumbId int64) (*documentData, error) { - data := &dataobject.DocumentsDO{ - DocumentId: documentId, - AccessHash: rand.Int63(), - DcId: 2, - FilePath: filePath, - FileSize: fileSize, - UploadedFileName: uploadedFileName, - Ext: extName, - MimeType: mimeType, - ThumbId: thumbId, - Version: 0, - } - data.Id = dao.GetDocumentsDAO(dao.DB_MASTER).Insert(data) - return &documentData{DocumentsDO: data}, nil -} - -func GetDocumentFileData(id, accessHash int64, version int32, offset, limit int32) (*mtproto.Upload_File, error) { - do := dao.GetDocumentsDAO(dao.DB_MASTER).SelectByFileLocation(id, accessHash, version) - if do == nil { - return nil, fmt.Errorf("not found document <%d, %d, %d>", id, accessHash, version) - } - - if offset > do.FileSize { - limit = 0 - } else if offset+limit > do.FileSize { - limit = do.FileSize - offset - } - - var filename = core.NBFS_DATA_PATH + do.FilePath - f, err := os.Open(filename) - if err != nil { - glog.Error("open ", filename, " error: ", err) - return nil, err - } - defer f.Close() - - bytes := make([]byte, limit) - _, err = f.ReadAt(bytes, int64(offset)) - if err != nil { - glog.Error("read file ", filename, " error: ", err) - return nil, err - } - - uploadFile := &mtproto.TLUploadFile{Data2: &mtproto.Upload_File_Data{ - Type: core.MakeStorageFileType(do.Ext), - Mtime: int32(time.Now().Unix()), - Bytes: bytes, - }} - return uploadFile.To_Upload_File(), nil -} - -func makeDocumentByDO(do *dataobject.DocumentsDO) *mtproto.Document { - var ( - thumb *mtproto.PhotoSize - document *mtproto.Document - ) - - if do == nil { - document = mtproto.NewTLDocumentEmpty().To_Document() - } else { - if do.ThumbId != 0 { - sizeList := photo.GetPhotoSizeList(do.ThumbId) - if len(sizeList) > 0 { - thumb = sizeList[0] - } - } - if thumb == nil { - thumb = mtproto.NewTLPhotoSizeEmpty().To_PhotoSize() - } - - attributes := &mtproto.DocumentAttributeList{} - err := json.Unmarshal([]byte(do.Attributes), attributes) - if err != nil { - glog.Error(err) - attributes.Attributes = []*mtproto.DocumentAttribute{} - } - - // if do.Attributes - document = &mtproto.Document{ - Constructor: mtproto.TLConstructor_CRC32_document, - Data2: &mtproto.Document_Data{ - Id: do.DocumentId, - AccessHash: do.AccessHash, - Date: int32(time.Now().Unix()), - MimeType: do.MimeType, - Size: do.FileSize, - Thumb: thumb, - DcId: 2, - Version: do.Version, - Attributes: attributes.Attributes, - }, - } - } - - return document -} - -func GetDocument(id, accessHash int64, version int32) *mtproto.Document { - do := dao.GetDocumentsDAO(dao.DB_SLAVE).SelectByFileLocation(id, accessHash, version) - if do == nil { - glog.Warning("") - } - return makeDocumentByDO(do) -} - -func GetDocumentList(idList []int64) []*mtproto.Document { - doList := dao.GetDocumentsDAO(dao.DB_SLAVE).SelectByIdList(idList) - documetList := make([]*mtproto.Document, len(doList)) - for i := 0; i < len(doList); i++ { - documetList[i] = makeDocumentByDO(&doList[i]) - } - return documetList -} - -//func MakeStickerDocuemnt(id, accessHash int64) *mtproto.Document { -// // -// thumb := &mtproto.TLPhotoSize{Data2: &mtproto.PhotoSize_Data{ -// Type: "m", -// Location: &mtproto.FileLocation{ -// Constructor: mtproto.TLConstructor_CRC32_fileLocation, -// Data2: &mtproto.FileLocation_Data{ -// DcId: 2, -// VolumeId: 226607193, -// LocalId: 35316, -// Secret: 2434071452955778983, -// }}, -// W: 128, -// H: 128, -// Size: 5648, -// }} -// -// document := &mtproto.TLDocument{Data2: &mtproto.Document_Data{ -// Id: id, -// AccessHash: accessHash, -// Date: int32(time.Now().Unix()), -// MimeType: "image/jpg", -// Size: 37634, -// Thumb: thumb.To_PhotoSize(), -// DcId: 2, -// Version: 0, -// Attributes: MakeDocumentAttributes(id, accessHash), -// }} -// -// return document.To_Document() -//} -// -//func MakeDocumentAttributes(id, accessHash int64) []*mtproto.DocumentAttribute { -// attributes := make([]*mtproto.DocumentAttribute, 0, 3) -// imageSize := &mtproto.TLDocumentAttributeImageSize{Data2: &mtproto.DocumentAttribute_Data{ -// W: 512, -// H: 512, -// }} -// attributes = append(attributes, imageSize.To_DocumentAttribute()) -// -// sticker := &mtproto.TLDocumentAttributeSticker{Data2: &mtproto.DocumentAttribute_Data{ -// Alt: "😂", -// Stickerset: &mtproto.InputStickerSet{ -// Constructor: mtproto.TLConstructor_CRC32_inputStickerSetID, -// Data2: &mtproto.InputStickerSet_Data{ -// Id: id, -// AccessHash: accessHash, -// }, -// }, -// MaskCoords: nil, -// }} -// attributes = append(attributes, sticker.To_DocumentAttribute()) -// -// fileName := &mtproto.TLDocumentAttributeFilename{Data2: &mtproto.DocumentAttribute_Data{ -// FileName: "sticker.wep", -// }} -// attributes = append(attributes, fileName.To_DocumentAttribute()) -// -// return attributes -//} diff --git a/server/nbfs/biz/core/file/file_logic.go b/server/nbfs/biz/core/file/file_logic.go deleted file mode 100644 index 25f17870..00000000 --- a/server/nbfs/biz/core/file/file_logic.go +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package file - -import ( - "fmt" - "github.com/nebulaim/telegramd/server/nbfs/biz/base" - "github.com/nebulaim/telegramd/server/nbfs/biz/core" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dao" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" - "math/rand" - "os" - "path" - "strings" - // "github.com/golang/glog" -) - -// inputFile#f52ff27f id:long parts:int name:string md5_checksum:string = InputFile; -// inputFileBig#fa4f0bb5 id:long parts:int name:string = InputFile; -type fileData struct { - *dataobject.FilesDO -} - -// TODO(@benqi): 是否要加去重字段？？ -func NewFileData(filePartId int64, filePath, uploadName string, fileSize int64, md5Checksum string) (*fileData, error) { - var fileId = base.NextSnowflakeId() - var ext = path.Ext(uploadName) - ext = strings.ToLower(ext) - data2 := &dataobject.FilesDO{ - FileId: fileId, - AccessHash: int64(rand.Uint64()), - FilePartId: filePartId, - FileSize: fileSize, - FilePath: fmt.Sprintf("/0/%d%s", fileId, ext), - Ext: ext, - Md5Checksum: md5Checksum, - UploadName: uploadName, - } - - //var oldpath = core.NBFS_DATA_PATH + filePath - //var onewpath = core.NBFS_DATA_PATH + data2.FilePath - // - //f, err := os.Create(onewpath) - //if err != nil { - // glog.Error(err) - // return nil, err - //} - //defer f.Close() - // - //for i := 0; i < - //_, err = f.Write(bytes) - //if err != nil { - // glog.Error(err) - // return err - //} - //f.Sync() - - // var oldpath = core.NBFS_DATA_PATH + filePath - // var newpath = - // os.Rename(core.NBFS_DATA_PATH + filePath) - data2.Id = dao.GetFilesDAO(dao.DB_MASTER).Insert(data2) - - err := os.Rename(core.NBFS_DATA_PATH+filePath, core.NBFS_DATA_PATH+data2.FilePath) - if err != nil { - return nil, err - } - return &fileData{FilesDO: data2}, nil -} - -func MakeFileDataByLoad(fileId, accessHash int64) (*fileData, error) { - data2 := dao.GetFilesDAO(dao.DB_SLAVE).Select(fileId) - if data2 == nil { - return nil, fmt.Errorf("not found file_id: %d", fileId) - } - - if data2.AccessHash != accessHash { - return nil, fmt.Errorf("invalid access_hash: %d", accessHash) - } - return &fileData{FilesDO: data2}, nil -} diff --git a/server/nbfs/biz/core/file/file_part_logic.go b/server/nbfs/biz/core/file/file_part_logic.go deleted file mode 100644 index 48e918ba..00000000 --- a/server/nbfs/biz/core/file/file_part_logic.go +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package file - -import ( - "crypto/md5" - "fmt" - "github.com/golang/glog" - base2 "github.com/nebulaim/telegramd/baselib/base" - "github.com/nebulaim/telegramd/server/nbfs/biz/base" - "github.com/nebulaim/telegramd/server/nbfs/biz/core" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dao" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" - "io/ioutil" - "os" -) - -const ( - kMaxFilePartSize = 65536 -) - -type filePartData struct { - SavedFilePath string - SavedMd5Hash string - *dataobject.FilePartsDO -} - -// 判断文件是否存在 -func pathExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return false, err -} - -func DoSavedFilePart(creatorId, filePartId int64) (*filePartData, error) { - data := dao.GetFilePartsDAO(dao.DB_MASTER).SelectFileParts(creatorId, filePartId) - if data == nil { - return nil, fmt.Errorf("not found file_parts by {creator_id: %d, file_id: %d}", creatorId, filePartId) - } - - // 1. calc md5, file_size, total_file_parts - md5Hash := md5.New() - fileSzie := 0 - // FilePath: - newFilePath := fmt.Sprintf("/0/%d.cache", base.NextSnowflakeId()) - newFileName := core.NBFS_DATA_PATH + newFilePath - f, err := os.Create(newFileName) - if err != nil { - glog.Error(err) - return nil, err - } - defer f.Close() - for i := int32(0); i <= data.FilePart; i++ { - fileName := fmt.Sprintf("%s%s/%d.part", core.NBFS_DATA_PATH, data.FilePath, i) - exist, err := pathExists(fileName) - if err != nil { - glog.Errorf("path - %s, error : %v", fileName, err) - return nil, err - } - if !exist { - err = fmt.Errorf("path not exists: %s", fileName) - return nil, err - } - - b, err := ioutil.ReadFile(fileName) - if err != nil { - err = fmt.Errorf("path not exists: %s", fileName) - return nil, err - } - - md5Hash.Write(b) - fileSzie += len(b) - _, err = f.Write(b) - if err != nil { - glog.Error(err) - return nil, err - } - } - f.Sync() - - data.FileSize = int64(fileSzie) - data.FileTotalParts = data.FilePart + 1 - data.FilePath = newFilePath - - dao.GetFilePartsDAO(dao.DB_MASTER).UpdateFilePartAndTotal(data.FilePart, data.FileTotalParts, data.FileSize, data.Id) - - return &filePartData{ - SavedFilePath: newFilePath, - SavedMd5Hash: fmt.Sprintf("%x", md5Hash.Sum(nil)), - FilePartsDO: data, - }, nil -} - -func MakeFilePartData(creatorId, filePartId int64, isNew, isBigFile bool) (*filePartData, error) { - var data *dataobject.FilePartsDO - data = dao.GetFilePartsDAO(dao.DB_MASTER).SelectFileParts(creatorId, filePartId) - if data == nil { - data = &dataobject.FilePartsDO{ - CreatorId: creatorId, - FilePartId: filePartId, - FilePart: -1, - IsBigFile: base2.BoolToInt8(isBigFile), - // FileTotalParts: fileTotalParts, - FilePath: fmt.Sprintf("/0/%d.parts", base.NextSnowflakeId()), - } - data.Id = dao.GetFilePartsDAO(dao.DB_MASTER).Insert(data) - } - //if isNew { - //} else { - // if data == nil { - // return nil, fmt.Errorf("not found file_parts by {creator_id: %d, file_id: %d}", creatorId, filePartId) - // } - //} - return &filePartData{FilePartsDO: data}, nil -} - -func saveFileData(filePath string, filePart int32, bytes []byte) error { - exist, err := pathExists(filePath) - if err != nil { - glog.Errorf("pathExists error![%v]", err) - return err - } - - if !exist { - err := os.Mkdir(filePath, 0755) - if err != nil { - glog.Errorf("mkdir failed![%v]\n", err) - return err - } - } - - fileName := fmt.Sprintf("%s/%d.part", filePath, filePart) - exist, _ = pathExists(fileName) - if !exist { - err = ioutil.WriteFile(fileName, bytes, 0644) - if err != nil { - glog.Error(err) - return err - } - } - - //f, err := os.Create(fileName) - //if err != nil { - // glog.Error(err) - // return err - //} - //defer f.Close() - //_, err = f.Write(bytes) - //if err != nil { - // glog.Error(err) - // return err - //} - //f.Sync() - - return nil -} - -func (m *filePartData) SaveFilePart(filePart int32, bytes []byte) error { - //if filePart - m.FilePart != 1 { - // return fmt.Errorf("bad request, err: {filePart: %d, m.FilePart: %d}", filePart, m.FilePart) - //} - - fileName := core.NBFS_DATA_PATH + m.FilePath - err := saveFileData(fileName, filePart, bytes) - if err != nil { - return err - } - - if m.FilePart < filePart { - dao.GetFilePartsDAO(dao.DB_MASTER).UpdateFilePart(filePart, m.Id) - } - - //var begin = filePart == 0 - //var end = len(bytes) < kMaxFilePartSize - // - //m.FilePart = filePart - //if end { - // m.FileTotalParts = filePart + 1 - // m.FileSize = int64(filePart)*kMaxFilePartSize + int64(len(bytes)) - //} - // - //if begin { - // dao.GetFilePartsDAO(dao.DB_MASTER).Insert(m.FilePartsDO) - //} else { - // if end { - // dao.GetFilePartsDAO(dao.DB_MASTER).UpdateFilePartAndTotal(filePart, m.FileTotalParts, m.FileSize, m.Id) - // } else { - // dao.GetFilePartsDAO(dao.DB_MASTER).UpdateFilePart(filePart, m.Id) - // } - //} - // - //if end { - // // 文件上传结束, 计算出文件大小和md5，存盘 - // var ( - // fileId = helper.NextSnowflakeId() - // accessHash = rand.Int63() - // ) - // - // if m.data.IsBigFile == 0 { - // // TODO(@benqi): md5优化 - // md5Checksum, _ := md5File(m.data.FilePath) - // return m.onUploadEnd(fileId, accessHash, md5Checksum) - // } else { - // return m.onUploadEnd(fileId, accessHash, "") - // } - //} - - return nil -} diff --git a/server/nbfs/biz/core/file/file_part_logic_test.go b/server/nbfs/biz/core/file/file_part_logic_test.go deleted file mode 100644 index cbd5f0d0..00000000 --- a/server/nbfs/biz/core/file/file_part_logic_test.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package file - -import ( - "fmt" - "github.com/nebulaim/telegramd/baselib/mysql_client" - "github.com/nebulaim/telegramd/server/nbfs/biz/core" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dao" - "io/ioutil" - "math/rand" - "testing" - "time" -) - -func init() { - rand.Seed(time.Now().UnixNano()) - mysqlConfig1 := mysql_client.MySQLConfig{ - Name: "immaster", - DSN: "root:@/nebulaim?charset=utf8", - Active: 5, - Idle: 2, - } - mysqlConfig2 := mysql_client.MySQLConfig{ - Name: "imslave", - DSN: "root:@/nebulaim?charset=utf8", - Active: 5, - Idle: 2, - } - mysql_client.InstallMysqlClientManager([]mysql_client.MySQLConfig{mysqlConfig1, mysqlConfig2}) - dao.InstallMysqlDAOManager(mysql_client.GetMysqlClientManager()) -} - -// go test -v -run=TestSaveFilePart -func TestSaveFilePart(t *testing.T) { - var uploadName = "./test002.jpeg" - buf, err := ioutil.ReadFile(uploadName) - if err != nil { - panic(err) - } - - sz := len(buf) / kMaxFilePartSize - - var creatorId = rand.Int63() - var filePartId = rand.Int63() - - var blockSize = kMaxFilePartSize - // = sz % kMaxFilePartSize - var uploadFileName string - - var filePart *filePartData - for i := 0; i <= sz; i++ { - var isNew = i == 0 - filePart, err = MakeFilePartData(creatorId, filePartId, isNew, false) - if i == sz { - blockSize = len(buf) % kMaxFilePartSize - uploadFileName = filePart.FilePath - } - - filePart.SaveFilePart(int32(i), buf[i*kMaxFilePartSize:i*kMaxFilePartSize+blockSize]) - fmt.Println(*filePart.FilePartsDO, ", file_part: ", i, ", block_size: ", blockSize) - } - - md5, _ := core.CalcMd5File(uploadName) - fileLogic, _ := NewFileData(filePartId, uploadFileName, uploadName, int64(len(buf)), md5) - _ = fileLogic -} diff --git a/server/nbfs/biz/core/photo/photo.go b/server/nbfs/biz/core/photo/photo.go deleted file mode 100644 index afcce218..00000000 --- a/server/nbfs/biz/core/photo/photo.go +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package photo - -import ( - // "github.com/nebulaim/telegramd/biz/dal/dao" - "github.com/nebulaim/telegramd/proto/mtproto" -) - -const ( - PHOTO_SIZE_ORIGINAL = "0" // client upload original photo - PHOTO_SIZE_SMALL_TYPE = "s" - PHOTO_SIZE_MEDIUMN_TYPE = "m" - PHOTO_SIZE_XLARGE_TYPE = "x" - PHOTO_SIZE_YLARGE_TYPE = "y" - PHOTO_SIZE_A_TYPE = "a" - PHOTO_SIZE_B_TYPE = "b" - PHOTO_SIZE_C_TYPE = "c" - - PHOTO_SIZE_SMALL_SIZE = 90 - PHOTO_SIZE_MEDIUMN_SIZE = 320 - PHOTO_SIZE_XLARGE_SIZE = 800 - PHOTO_SIZE_YLARGE_SIZE = 1280 - PHOTO_SIZE_A_SIZE = 160 - PHOTO_SIZE_B_SIZE = 320 - PHOTO_SIZE_C_SIZE = 640 -) - -/* - inputFile#f52ff27f id:long parts:int name:string md5_checksum:string = InputFile; - inputFileBig#fa4f0bb5 id:long parts:int name:string = InputFile; - - inputPhotoEmpty#1cd7bf0d = InputPhoto; - inputPhoto#fb95c6c4 id:long access_hash:long = InputPhoto; - - photoEmpty#2331b22d id:long = Photo; - photo#9288dd29 flags:# has_stickers:flags.0?true id:long access_hash:long date:int sizes:Vector = Photo; - - fileLocationUnavailable#7c596b46 volume_id:long local_id:int secret:long = FileLocation; - fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation; - - photoSizeEmpty#e17e23c type:string = PhotoSize; - photoSize#77bfb61b type:string location:FileLocation w:int h:int size:int = PhotoSize; - photoCachedSize#e9a734fa type:string location:FileLocation w:int h:int bytes:bytes = PhotoSize; - - - photos.photos#8dca6aa5 photos:Vector users:Vector = photos.Photos; - photos.photosSlice#15051f54 count:int photos:Vector users:Vector = photos.Photos; - - photos.photo#20212ca8 photo:Photo users:Vector = photos.Photo; - - upload.file#96a18d5 type:storage.FileType mtime:int bytes:bytes = upload.File; - upload.fileCdnRedirect#ea52fe5a dc_id:int file_token:bytes encryption_key:bytes encryption_iv:bytes cdn_file_hashes:Vector = upload.File; - - photos.updateProfilePhoto#f0bb5152 id:InputPhoto = UserProfilePhoto; - photos.uploadProfilePhoto#4f32c098 file:InputFile = photos.Photo; - photos.deletePhotos#87cf7f2f id:Vector = Vector; - photos.getUserPhotos#91cd32a8 user_id:InputUser offset:int max_id:long limit:int = photos.Photos; - - userProfilePhotoEmpty#4f11bae1 = UserProfilePhoto; - userProfilePhoto#d559d8c8 photo_id:long photo_small:FileLocation photo_big:FileLocation = UserProfilePhoto; -*/ - -/* - storage.fileUnknown#aa963b05 = storage.FileType; - storage.filePartial#40bc6f52 = storage.FileType; - storage.fileJpeg#7efe0e = storage.FileType; - storage.fileGif#cae1aadf = storage.FileType; - storage.filePng#a4f63c0 = storage.FileType; - storage.filePdf#ae1e508d = storage.FileType; - storage.fileMp3#528a0677 = storage.FileType; - storage.fileMov#4b09ebbc = storage.FileType; - storage.fileMp4#b3cea0e4 = storage.FileType; - storage.fileWebp#1081464c = storage.FileType; -*/ - -//var sizeList = []int{ -// PHOTO_SIZE_SMALL_SIZE, -// PHOTO_SIZE_SMALL_SIZE, -// PHOTO_SIZE_MEDIUMN_SIZE, -// PHOTO_SIZE_XLARGE_SIZE, -// PHOTO_SIZE_YLARGE_SIZE, -// PHOTO_SIZE_A_SIZE, -// PHOTO_SIZE_B_SIZE, -// PHOTO_SIZE_C_SIZE, -//} -// -//func getSizeType(idx int) string { -// switch idx { -// case 0: -// return PHOTO_SIZE_SMALL_TYPE -// case 1: -// return PHOTO_SIZE_MEDIUMN_TYPE -// case 2: -// return PHOTO_SIZE_XLARGE_TYPE -// case 3: -// return PHOTO_SIZE_YLARGE_TYPE -// case 4: -// return PHOTO_SIZE_A_TYPE -// case 5: -// return PHOTO_SIZE_B_TYPE -// case 6: -// return PHOTO_SIZE_C_TYPE -// } -// -// return "" -//} -// -//type resizeInfo struct { -// isWidth bool -// size int -//} -// -//func MakeResizeInfo(img image.Image) resizeInfo { -// w := img.Bounds().Dx() -// h := img.Bounds().Dy() -// -// if w >= h { -// return resizeInfo{ -// isWidth: true, -// size: w, -// } -// } else { -// return resizeInfo{ -// isWidth: false, -// size : h, -// } -// } -//} - -////////////////////////////////////////////////////////////////////////////// -//func GetPhotoSizeList(photoId int64) (sizes []*mtproto.PhotoSize) { -// doList := dao.GetPhotoDatasDAO(dao.DB_SLAVE).SelectListByPhotoId(photoId) -// sizes = make([]*mtproto.PhotoSize, 0, len(doList)) -// for _, do := range doList { -// sizeData := &mtproto.PhotoSize_Data{ -// Type: getSizeType(int(do.LocalId)), -// W: do.Width, -// H: do.Height, -// Size: int32(len(do.Bytes)), -// Location: &mtproto.FileLocation{ -// Constructor: mtproto.TLConstructor_CRC32_fileLocation, -// Data2: &mtproto.FileLocation_Data{ -// VolumeId: do.VolumeId, -// LocalId: int32(do.LocalId), -// Secret: do.AccessHash, -// DcId: do.DcId, -// }, -// }, -// } -// -// if do.LocalId== 0 { -// sizes = append(sizes, &mtproto.PhotoSize{ -// Constructor: mtproto.TLConstructor_CRC32_photoCachedSize, -// Data2: sizeData,}) -// sizeData.Bytes = do.Bytes -// } else { -// sizes = append(sizes, &mtproto.PhotoSize{ -// Constructor: mtproto.TLConstructor_CRC32_photoSize, -// Data2: sizeData,}) -// } -// } -// return -//} - -//// TODO(@benqi): -//// 我们未来的图片存储系统可能会按facebook的Haystack论文来实现 -//// mtproto协议也定义了一套自己的文件存储方案，fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation; -//// 在这里，我们重新定义mtproto的volume_id和local_id，对应Haystack的key和alternate_key，secret对应cookie -//// 在当前简单实现里，volume_id由sonwflake生成，local_id对应于图片类型，secret为access_hash -//// TODO(@benqi): -//// 参数使用mtproto.File -//func UploadPhoto(userId int32, photoId, fileId int64, parts int32, name, md5Checksum string) ([]*mtproto.PhotoSize, error) { -// sizes := make([]*mtproto.PhotoSize, 0, 4) -// -// // 图片压缩和处理 -// // ext := filepath.Ext(name) -// -// filesDO := dao.GetFilesDAO(dao.DB_MASTER).SelectByIDAndParts(fileId, parts) -// if filesDO == nil { -// return nil, fmt.Errorf("File exists: id = %d, parts = %d", fileId, parts) -// } -// -// // check md5Checksum, big file's md5 is empty -// if md5Checksum != "" && md5Checksum != filesDO.Md5Checksum { -// return nil, fmt.Errorf("Invalid md5Checksum: md5Checksum = %s, but filesDO = {%v}", md5Checksum, filesDO) -// } -// -// // select file data -// filePartsDOList := dao.GetFilePartsDAO(dao.DB_MASTER).SelectFileParts(fileId) -// fileDatas := []byte{} -// -// for _, p := range filePartsDOList { -// fileDatas = append(fileDatas, p.Bytes...) -// } -// -// // bufio.Reader{} -// img, err := imaging.Decode(bytes.NewReader(fileDatas)) -// if err != nil { -// glog.Errorf("Decode error: {%v}", err) -// return nil, err -// } -// -// imgSz := MakeResizeInfo(img) -// -// vId := base2.NextSnowflakeId() -// for i, sz := range sizeList { -// photoDatasDO := &dataobject.PhotoDatasDO{ -// PhotoId: photoId, -// DcId: 2, -// VolumeId: vId, -// LocalId: int32(i), -// AccessHash: rand.Int63(), -// } -// -// var dst *image.NRGBA -// if imgSz.isWidth { -// dst = imaging.Resize(img, sz, 0, imaging.Lanczos) -// } else { -// dst = imaging.Resize(img, 0, sz, imaging.Lanczos) -// } -// -// photoDatasDO.Width = int32(dst.Bounds().Dx()) -// photoDatasDO.Height = int32(dst.Bounds().Dy()) -// imgBuf := helper.MakeBuffer(0, len(fileDatas)) -// err = imaging.Encode(imgBuf, dst, imaging.JPEG) -// if err != nil { -// glog.Errorf("Encode error: {%v}", err) -// return nil, err -// } -// -// photoDatasDO.Bytes = imgBuf.Bytes() -// dao.GetPhotoDatasDAO(dao.DB_MASTER).Insert(photoDatasDO) -// -// photoSizeData := &mtproto.PhotoSize_Data{ -// Type: getSizeType(i), -// W: photoDatasDO.Width, -// H: photoDatasDO.Height, -// Size: int32(len(photoDatasDO.Bytes)), -// Location: &mtproto.FileLocation{ -// Constructor: mtproto.TLConstructor_CRC32_fileLocation, -// Data2: &mtproto.FileLocation_Data{ -// VolumeId: photoDatasDO.VolumeId, -// LocalId: int32(i), -// Secret: photoDatasDO.AccessHash, -// DcId: photoDatasDO.DcId}}} -// -// if i== 0 { -// sizes = append(sizes, &mtproto.PhotoSize{ -// Constructor: mtproto.TLConstructor_CRC32_photoCachedSize, -// Data2: photoSizeData,}) -// photoSizeData.Bytes = photoDatasDO.Bytes -// } else { -// sizes = append(sizes, &mtproto.PhotoSize{ -// Constructor: mtproto.TLConstructor_CRC32_photoSize, -// Data2: photoSizeData,}) -// } -// } -// -// return sizes, nil -//} -// -//func GetFileAccessHash(fileId int64, fileParts int32) int64 { -// do := dao.GetFilesDAO(dao.DB_MASTER).SelectByIDAndParts(fileId, fileParts) -// if do == nil { -// return 0 -// } else { -// return do.AccessHash -// } -//} - -func MakeUserProfilePhoto(photoId int64, sizes []*mtproto.PhotoSize) *mtproto.UserProfilePhoto { - if len(sizes) == 0 { - return mtproto.NewTLUserProfilePhotoEmpty().To_UserProfilePhoto() - } - - // TODO(@benqi): check PhotoSize is photoSizeEmpty - photo := &mtproto.TLUserProfilePhoto{Data2: &mtproto.UserProfilePhoto_Data{ - PhotoId: photoId, - PhotoSmall: sizes[0].GetData2().GetLocation(), - PhotoBig: sizes[len(sizes)-1].GetData2().GetLocation(), - }} - - return photo.To_UserProfilePhoto() -} - -func MakeChatPhoto(sizes []*mtproto.PhotoSize) *mtproto.ChatPhoto { - if len(sizes) == 0 { - return mtproto.NewTLChatPhotoEmpty().To_ChatPhoto() - } - - // TODO(@benqi): check PhotoSize is photoSizeEmpty - photo := &mtproto.TLChatPhoto{Data2: &mtproto.ChatPhoto_Data{ - PhotoSmall: sizes[0].GetData2().GetLocation(), - PhotoBig: sizes[len(sizes)-1].GetData2().GetLocation(), - }} - - return photo.To_ChatPhoto() -} diff --git a/server/nbfs/biz/core/photo/photo_file.go b/server/nbfs/biz/core/photo/photo_file.go deleted file mode 100644 index 564eade4..00000000 --- a/server/nbfs/biz/core/photo/photo_file.go +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package photo - -import ( - "fmt" - "github.com/disintegration/imaging" - "github.com/golang/glog" - "github.com/nebulaim/telegramd/proto/mtproto" - base2 "github.com/nebulaim/telegramd/server/nbfs/biz/base" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" - "image" - "math/rand" - // "os" - "github.com/nebulaim/telegramd/server/nbfs/biz/core" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dao" - "io/ioutil" - "os" - "time" -) - -const ( - kPhotoSizeOriginalType = "0" // client upload original photo - kPhotoSizeSmallType = "s" - kPhotoSizeMediumType = "m" - kPhotoSizeXLargeType = "x" - kPhotoSizeYLargeType = "y" - kPhotoSizeAType = "a" - kPhotoSizeBType = "b" - kPhotoSizeCType = "c" - - kPhotoSizeOriginalSize = 0 // client upload original photo - kPhotoSizeSmallSize = 90 - kPhotoSizeMediumSize = 320 - kPhotoSizeXLargeSize = 800 - kPhotoSizeYLargeSize = 1280 - kPhotoSizeASize = 160 - kPhotoSizeBSize = 320 - kPhotoSizeCSize = 640 - - kPhotoSizeAIndex = 4 -) - -var sizeList = []int{ - kPhotoSizeOriginalSize, - kPhotoSizeSmallSize, - kPhotoSizeMediumSize, - kPhotoSizeXLargeSize, - kPhotoSizeYLargeSize, - kPhotoSizeASize, - kPhotoSizeBSize, - kPhotoSizeCSize, -} - -func getSizeType(idx int) string { - switch idx { - case 0: - return kPhotoSizeOriginalType - case 1: - return kPhotoSizeSmallType - case 2: - return kPhotoSizeMediumType - case 3: - return kPhotoSizeXLargeType - case 4: - return kPhotoSizeYLargeType - case 5: - return kPhotoSizeAType - case 6: - return kPhotoSizeBType - case 7: - return kPhotoSizeCType - } - - return "" -} - -//storage.fileUnknown#aa963b05 = storage.FileType; -//storage.filePartial#40bc6f52 = storage.FileType; -//storage.fileJpeg#7efe0e = storage.FileType; -//storage.fileGif#cae1aadf = storage.FileType; -//storage.filePng#a4f63c0 = storage.FileType; -//storage.filePdf#ae1e508d = storage.FileType; -//storage.fileMp3#528a0677 = storage.FileType; -//storage.fileMov#4b09ebbc = storage.FileType; -//storage.fileMp4#b3cea0e4 = storage.FileType; -//storage.fileWebp#1081464c = storage.FileType; - -//func checkIsABC(idx int) bool { -// if idx == -//} - -//import "github.com/golang/glog" -// - -type resizeInfo struct { - isWidth bool - size int -} - -func makeResizeInfo(img image.Image) resizeInfo { - w := img.Bounds().Dx() - h := img.Bounds().Dy() - - if w >= h { - return resizeInfo{ - isWidth: true, - size: w, - } - } else { - return resizeInfo{ - isWidth: false, - size: h, - } - } -} - -func getFileSize(filename string) int32 { - fi, e := os.Stat(filename) - if e != nil { - return 0 - } - // get the size - return int32(fi.Size()) -} - -// TODO(@benqi): -// 我们未来的图片存储系统可能会按facebook的Haystack论文来实现 -// mtproto协议也定义了一套自己的文件存储方案，fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation; -// 在这里，我们重新定义mtproto的volume_id和local_id，对应Haystack的key和alternate_key，secret对应cookie -// 在当前简单实现里，volume_id由sonwflake生成，local_id对应于图片类型，secret为access_hash -// TODO(@benqi): -// 参数使用mtproto.File -func UploadPhotoFile(photoId, accessHash int64, filePath, extName string, isABC bool) ([]*mtproto.PhotoSize, error) { - fileName := core.NBFS_DATA_PATH + filePath - - f, err := os.Open(fileName) - if err != nil { - glog.Error(err) - return nil, err - } - - defer f.Close() - - sizes := make([]*mtproto.PhotoSize, 0, 4) - - //fileDatas, err := ioutil.ReadFile(fileName) - //if err != nil { - // glog.Error(err) - // return nil, err - //} - - // bufio.Reader{} - img, err := imaging.Decode(f) - // img, err := imaging.Decode(bytes.NewReader(fileDatas)) - if err != nil { - glog.Errorf("Decode %s error: {%v}", fileName, err) - return nil, err - } - - imgSz := makeResizeInfo(img) - - vId := base2.NextSnowflakeId() - for i, sz := range sizeList { - if i != 0 { - if isABC { - if i <= kPhotoSizeAIndex { - continue - } - } else { - if i > kPhotoSizeAIndex { - continue - } - } - } - photoDatasDO := &dataobject.PhotoDatasDO{ - PhotoId: photoId, - DcId: 2, - VolumeId: vId, - LocalId: int32(i), - AccessHash: rand.Int63(), - // Bytes: []byte{0}, - Ext: extName, - } - - if i == 0 { - photoDatasDO.Width = int32(img.Bounds().Dx()) - photoDatasDO.Height = int32(img.Bounds().Dy()) - photoDatasDO.FileSize = getFileSize(fileName) - photoDatasDO.FilePath = filePath - photoDatasDO.AccessHash = accessHash - } else { - var dst *image.NRGBA - if imgSz.isWidth { - dst = imaging.Resize(img, sz, 0, imaging.Lanczos) - } else { - dst = imaging.Resize(img, 0, sz, imaging.Lanczos) - } - - photoDatasDO.Width = int32(dst.Bounds().Dx()) - photoDatasDO.Height = int32(dst.Bounds().Dy()) - - // imgBuf := base2.MakeBuffer(0, len(fileDatas)) - photoDatasDO.FilePath = fmt.Sprintf("/%s/%d%s", getSizeType(i), vId, extName) - dstFileName := core.NBFS_DATA_PATH + photoDatasDO.FilePath - // fmt.Sprintf("%s/%s/%d%s", core.NBFS_DATA_PATH, getSizeType(i), vId, extName) - err = imaging.Save(dst, dstFileName) - // .Encode(imgBuf, dst, imaging.JPEG) - if err != nil { - glog.Errorf("Encode error: {%v}", err) - return nil, err - } - photoDatasDO.FileSize = getFileSize(dstFileName) - } - - // photoDatasDO.Bytes = imgBuf.Bytes() - dao.GetPhotoDatasDAO(dao.DB_MASTER).Insert(photoDatasDO) - - photoSizeData := &mtproto.PhotoSize_Data{ - Type: getSizeType(i), - W: photoDatasDO.Width, - H: photoDatasDO.Height, - // Size: int32(len(photoDatasDO.Bytes)), - Size: photoDatasDO.FileSize, - Location: &mtproto.FileLocation{ - Constructor: mtproto.TLConstructor_CRC32_fileLocation, - Data2: &mtproto.FileLocation_Data{ - VolumeId: photoDatasDO.VolumeId, - LocalId: int32(i), - Secret: photoDatasDO.AccessHash, - DcId: photoDatasDO.DcId}}} - - if i == 0 { - continue - } else if i == 1 { - sizes = append(sizes, &mtproto.PhotoSize{ - Constructor: mtproto.TLConstructor_CRC32_photoCachedSize, - Data2: photoSizeData}) - // TODO(@benqi): 如上预先存起来 - photoSizeData.Bytes, _ = ioutil.ReadFile(core.NBFS_DATA_PATH + photoDatasDO.FilePath) - // photoDatasDO.Bytes - } else { - sizes = append(sizes, &mtproto.PhotoSize{ - Constructor: mtproto.TLConstructor_CRC32_photoSize, - Data2: photoSizeData}) - } - } - - return sizes, nil -} - -//////////////////////////////////////////////////////////////////////////// -func GetPhotoSizeList(photoId int64) (sizes []*mtproto.PhotoSize) { - doList := dao.GetPhotoDatasDAO(dao.DB_SLAVE).SelectListByPhotoId(photoId) - sizes = make([]*mtproto.PhotoSize, 0, len(doList)) - for i := 1; i < len(doList); i++ { - sizeData := &mtproto.PhotoSize_Data{ - Type: getSizeType(int(doList[i].LocalId)), - W: doList[i].Width, - H: doList[i].Height, - Size: doList[i].FileSize, - Location: &mtproto.FileLocation{ - Constructor: mtproto.TLConstructor_CRC32_fileLocation, - Data2: &mtproto.FileLocation_Data{ - VolumeId: doList[i].VolumeId, - LocalId: int32(doList[i].LocalId), - Secret: doList[i].AccessHash, - DcId: doList[i].DcId, - }, - }, - } - - if i == 1 { - var filename = core.NBFS_DATA_PATH + doList[i].FilePath - cacheData, err := ioutil.ReadFile(filename) - if err != nil { - glog.Errorf("read file %s error: %v", filename, err) - sizeData.Bytes = []byte{} - } else { - sizeData.Bytes = cacheData - } - sizes = append(sizes, &mtproto.PhotoSize{ - Constructor: mtproto.TLConstructor_CRC32_photoCachedSize, - Data2: sizeData, - }) - } else { - sizes = append(sizes, &mtproto.PhotoSize{ - Constructor: mtproto.TLConstructor_CRC32_photoSize, - Data2: sizeData, - }) - } - } - return -} - -func GetPhotoFileData(volumeId int64, localId int32, secret int64, offset int32, limit int32) (*mtproto.Upload_File, error) { - // inputFileLocation#14637196 volume_id:long local_id:int secret:long = InputFileLocation; - do := dao.GetPhotoDatasDAO(dao.DB_MASTER).SelectByFileLocation(volumeId, localId, secret) - if do == nil { - return nil, fmt.Errorf("not found photo <%d, %d, %d>", volumeId, localId, secret) - } - - if offset > do.FileSize { - limit = 0 - } else if offset+limit > do.FileSize { - limit = do.FileSize - offset - } - - var filename = core.NBFS_DATA_PATH + do.FilePath - f, err := os.Open(filename) - if err != nil { - glog.Error("open ", filename, " error: ", err) - return nil, err - } - defer f.Close() - - bytes := make([]byte, limit) - _, err = f.ReadAt(bytes, int64(offset)) - if err != nil { - glog.Error("read file ", filename, " error: ", err) - return nil, err - } - - uploadFile := &mtproto.TLUploadFile{Data2: &mtproto.Upload_File_Data{ - Type: core.MakeStorageFileType(do.Ext), - Mtime: int32(time.Now().Unix()), - Bytes: bytes, - }} - return uploadFile.To_Upload_File(), nil -} diff --git a/server/nbfs/biz/core/photo/photo_test.go b/server/nbfs/biz/core/photo/photo_test.go deleted file mode 100644 index 1751c098..00000000 --- a/server/nbfs/biz/core/photo/photo_test.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package photo - -import ( - "testing" - //"fmt" - //"github.com/disintegration/imaging" - //"bytes" - "github.com/nebulaim/telegramd/baselib/mysql_client" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dao" - //"image" -) - -func init() { - mysqlConfig := mysql_client.MySQLConfig{ - Name: "immaster", - DSN: "root:@/nebulaim?charset=utf8", - Active: 5, - Idle: 2, - } - mysql_client.InstallMysqlClientManager([]mysql_client.MySQLConfig{mysqlConfig}) - dao.InstallMysqlDAOManager(mysql_client.GetMysqlClientManager()) -} - -func TestResize(t *testing.T) { - //id := int64(-8540733062663239681) - //filePartsDO := dao.GetFilePartsDAO(dao.DB_MASTER).SelectFileParts(1, id) - ////fileDatas := []byte{} - ////for _, p := range filePartsDOList { - //// fileDatas = append(fileDatas, p.Bytes...) - ////} - // - //// bufio.Reader{} - //img, err := imaging.Decode(bytes.NewReader(fileDatas)) - //if err != nil { - // fmt.Printf("Decode error: {%v}\n", err) - // return - //} - // - //imgSz := makeResizeInfo(img) - //for i, sz := range sizeList { - // var dst *image.NRGBA - // if imgSz.isWidth { - // dst = imaging.Resize(img, sz, 0, imaging.Lanczos) - // } else { - // dst = imaging.Resize(img, 0, sz, imaging.Lanczos) - // } - // - // err := imaging.Save(dst, fmt.Sprintf("/tmp/telegramd/%d.jpg", i)) - // if err != nil { - // fmt.Printf("Save error: {%v}\n", err) - // } - //} -} diff --git a/server/nbfs/biz/dal/dao/mysql_dao/file_parts_dao.go b/server/nbfs/biz/dal/dao/mysql_dao/file_parts_dao.go deleted file mode 100644 index 571826db..00000000 --- a/server/nbfs/biz/dal/dao/mysql_dao/file_parts_dao.go +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package mysql_dao - -import ( - "fmt" - "github.com/golang/glog" - "github.com/jmoiron/sqlx" - "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" -) - -type FilePartsDAO struct { - db *sqlx.DB -} - -func NewFilePartsDAO(db *sqlx.DB) *FilePartsDAO { - return &FilePartsDAO{db} -} - -// insert into file_parts(creator_id, file_part_id, file_part, is_big_file, file_total_parts, file_path, file_size) values (:creator_id, :file_part_id, :file_part, :is_big_file, :file_total_parts, :file_path, :file_size) -// TODO(@benqi): sqlmap -func (dao *FilePartsDAO) Insert(do *dataobject.FilePartsDO) int64 { - var query = "insert into file_parts(creator_id, file_part_id, file_part, is_big_file, file_total_parts, file_path, file_size) values (:creator_id, :file_part_id, :file_part, :is_big_file, :file_total_parts, :file_path, :file_size)" - r, err := dao.db.NamedExec(query, do) - if err != nil { - errDesc := fmt.Sprintf("NamedExec in Insert(%v), error: %v", do, err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - id, err := r.LastInsertId() - if err != nil { - errDesc := fmt.Sprintf("LastInsertId in Insert(%v)_error: %v", do, err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - return id -} - -// select id, creator_id, file_part_id, file_part, is_big_file, file_total_parts, file_path, file_size from file_parts where creator_id = :creator_id and file_part_id = :file_part_id -// TODO(@benqi): sqlmap -func (dao *FilePartsDAO) SelectFileParts(creator_id int64, file_part_id int64) *dataobject.FilePartsDO { - var query = "select id, creator_id, file_part_id, file_part, is_big_file, file_total_parts, file_path, file_size from file_parts where creator_id = ? and file_part_id = ?" - rows, err := dao.db.Queryx(query, creator_id, file_part_id) - - if err != nil { - errDesc := fmt.Sprintf("Queryx in SelectFileParts(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - defer rows.Close() - - do := &dataobject.FilePartsDO{} - if rows.Next() { - err = rows.StructScan(do) - if err != nil { - errDesc := fmt.Sprintf("StructScan in SelectFileParts(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - } else { - return nil - } - - err = rows.Err() - if err != nil { - errDesc := fmt.Sprintf("rows in SelectFileParts(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - return do -} - -// update file_parts set file_part = :file_part where id = :id -// TODO(@benqi): sqlmap -func (dao *FilePartsDAO) UpdateFilePart(file_part int32, id int64) int64 { - var query = "update file_parts set file_part = ? where id = ?" - r, err := dao.db.Exec(query, file_part, id) - - if err != nil { - errDesc := fmt.Sprintf("Exec in UpdateFilePart(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - rows, err := r.RowsAffected() - if err != nil { - errDesc := fmt.Sprintf("RowsAffected in UpdateFilePart(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - return rows -} - -// update file_parts set file_part = :file_part, file_total_parts = :file_total_parts, file_size = :file_size where id = :id -// TODO(@benqi): sqlmap -func (dao *FilePartsDAO) UpdateFilePartAndTotal(file_part int32, file_total_parts int32, file_size int64, id int64) int64 { - var query = "update file_parts set file_part = ?, file_total_parts = ?, file_size = ? where id = ?" - r, err := dao.db.Exec(query, file_part, file_total_parts, file_size, id) - - if err != nil { - errDesc := fmt.Sprintf("Exec in UpdateFilePartAndTotal(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - rows, err := r.RowsAffected() - if err != nil { - errDesc := fmt.Sprintf("RowsAffected in UpdateFilePartAndTotal(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - return rows -} diff --git a/server/nbfs/biz/dal/dao/mysql_dao/files_dao.go b/server/nbfs/biz/dal/dao/mysql_dao/files_dao.go deleted file mode 100644 index 736e9d95..00000000 --- a/server/nbfs/biz/dal/dao/mysql_dao/files_dao.go +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package mysql_dao - -import ( - "fmt" - "github.com/golang/glog" - "github.com/jmoiron/sqlx" - "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" -) - -type FilesDAO struct { - db *sqlx.DB -} - -func NewFilesDAO(db *sqlx.DB) *FilesDAO { - return &FilesDAO{db} -} - -// insert into files(file_id, access_hash, file_size, file_path, ext, md5_checksum, upload_name) values (:file_id, :access_hash, :file_size, :file_path, :ext, :md5_checksum, :upload_name) -// TODO(@benqi): sqlmap -func (dao *FilesDAO) Insert(do *dataobject.FilesDO) int64 { - var query = "insert into files(file_id, access_hash, file_size, file_path, ext, md5_checksum, upload_name) values (:file_id, :access_hash, :file_size, :file_path, :ext, :md5_checksum, :upload_name)" - r, err := dao.db.NamedExec(query, do) - if err != nil { - errDesc := fmt.Sprintf("NamedExec in Insert(%v), error: %v", do, err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - id, err := r.LastInsertId() - if err != nil { - errDesc := fmt.Sprintf("LastInsertId in Insert(%v)_error: %v", do, err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - return id -} - -// select id, file_id, access_hash, file_size, file_path, ext, md5_checksum, upload_name from files where file_id = :file_id -// TODO(@benqi): sqlmap -func (dao *FilesDAO) Select(file_id int64) *dataobject.FilesDO { - var query = "select id, file_id, access_hash, file_size, file_path, ext, md5_checksum, upload_name from files where file_id = ?" - rows, err := dao.db.Queryx(query, file_id) - - if err != nil { - errDesc := fmt.Sprintf("Queryx in Select(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - defer rows.Close() - - do := &dataobject.FilesDO{} - if rows.Next() { - err = rows.StructScan(do) - if err != nil { - errDesc := fmt.Sprintf("StructScan in Select(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - } else { - return nil - } - - err = rows.Err() - if err != nil { - errDesc := fmt.Sprintf("rows in Select(_), error: %v", err) - glog.Error(errDesc) - panic(mtproto.NewRpcError(int32(mtproto.TLRpcErrorCodes_DBERR), errDesc)) - } - - return do -} diff --git a/server/nbfs/biz/dal/dataobject/files_do.go b/server/nbfs/biz/dal/dataobject/files_do.go deleted file mode 100644 index a52740df..00000000 --- a/server/nbfs/biz/dal/dataobject/files_do.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dataobject - -type FilesDO struct { - Id int64 `db:"id"` - FileId int64 `db:"file_id"` - AccessHash int64 `db:"access_hash"` - CreatorId int64 `db:"creator_id"` - CreatorUserId int32 `db:"creator_user_id"` - FilePartId int64 `db:"file_part_id"` - FileParts int32 `db:"file_parts"` - FileSize int64 `db:"file_size"` - FilePath string `db:"file_path"` - Ext string `db:"ext"` - IsBigFile int8 `db:"is_big_file"` - Md5Checksum string `db:"md5_checksum"` - UploadName string `db:"upload_name"` - CreatedAt string `db:"created_at"` -} diff --git a/server/nbfs/biz/dal/tables/file_parts.xml b/server/nbfs/biz/dal/tables/file_parts.xml deleted file mode 100644 index 75383b37..00000000 --- a/server/nbfs/biz/dal/tables/file_parts.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - INSERT INTO file_parts - (creator_id, file_part_id, file_part, is_big_file, file_total_parts, file_path, file_size) - VALUES - (:creator_id, :file_part_id, :file_part, :is_big_file, :file_total_parts, :file_path, :file_size) - - - - - SELECT - id, creator_id, file_part_id, file_part, is_big_file, file_total_parts, file_path, file_size - FROM - file_parts - WHERE - creator_id = :creator_id AND file_part_id = :file_part_id - - - - - UPDATE file_parts SET file_part = :file_part WHERE id = :id - - - - - UPDATE file_parts SET file_part = :file_part, file_total_parts = :file_total_parts, file_size = :file_size WHERE id = :id - - -
diff --git a/server/nbfs/biz/dal/tables/files.xml b/server/nbfs/biz/dal/tables/files.xml deleted file mode 100644 index 4288763f..00000000 --- a/server/nbfs/biz/dal/tables/files.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - INSERT INTO files - (file_id, access_hash, file_size, file_path, ext, md5_checksum, upload_name) - VALUES - (:file_id, :access_hash, :file_size, :file_path, :ext, :md5_checksum, :upload_name) - - - - - - SELECT - id, file_id, access_hash, file_size, file_path, ext, md5_checksum, upload_name - FROM - files - WHERE - file_id = :file_id - - -
diff --git a/server/nbfs/nbfs.go b/server/nbfs/nbfs.go deleted file mode 100644 index 66a6fa57..00000000 --- a/server/nbfs/nbfs.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "flag" - "github.com/golang/glog" - - "fmt" - "github.com/BurntSushi/toml" - "github.com/nebulaim/telegramd/baselib/grpc_util" - "github.com/nebulaim/telegramd/baselib/grpc_util/service_discovery" - "github.com/nebulaim/telegramd/baselib/mysql_client" - "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/core" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dao" - photo "github.com/nebulaim/telegramd/server/nbfs/photo/rpc" - upload "github.com/nebulaim/telegramd/server/nbfs/upload/rpc" - "google.golang.org/grpc" -) - -func init() { - flag.Set("alsologtostderr", "true") - flag.Set("log_dir", "false") -} - -type RpcServerConfig struct { - Addr string -} - -type NbfsConfig struct { - DataPath string -} - -type uploadServerConfig struct { - Nbfs *NbfsConfig - Server *RpcServerConfig - Discovery service_discovery.ServiceDiscoveryServerConfig - Mysql []mysql_client.MySQLConfig -} - -// 整合各服务，方便开发调试 -func main() { - flag.Parse() - - config := &uploadServerConfig{} - if _, err := toml.DecodeFile("./nbfs.toml", config); err != nil { - fmt.Errorf("%s\n", err) - return - } - - glog.Info(config.Nbfs, ", ", config.Server, ", ", config.Discovery, ", ", config.Mysql) - - // Init - core.InitNbfsDataPath(config.Nbfs.DataPath) - - // 初始化mysql_client、redis_client - // redis_client.InstallRedisClientManager(config.Redis) - mysql_client.InstallMysqlClientManager(config.Mysql) - - // 初始化redis_dao、mysql_dao - dao.InstallMysqlDAOManager(mysql_client.GetMysqlClientManager()) - // dao.InstallRedisDAOManager(redis_client.GetRedisClientManager()) - - // Start server - grpcServer := grpc_util.NewRpcServer(config.Server.Addr, &config.Discovery) - grpcServer.Serve(func(s *grpc.Server) { - mtproto.RegisterRPCUploadServer(s, &upload.UploadServiceImpl{DataPath: config.Nbfs.DataPath}) - mtproto.RegisterRPCNbfsServer(s, &photo.PhotoServiceImpl{}) - }) -} diff --git a/server/nbfs/photo/rpc/photo_service_impl.go b/server/nbfs/photo/rpc/photo_service_impl.go deleted file mode 100644 index ae3df9f1..00000000 --- a/server/nbfs/photo/rpc/photo_service_impl.go +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (c) 2018, https://github.com/nebulaim - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rpc - -import ( - "context" - "fmt" - "github.com/golang/glog" - "github.com/nebulaim/telegramd/baselib/logger" - "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/base" - document2 "github.com/nebulaim/telegramd/server/nbfs/biz/core/document" - "github.com/nebulaim/telegramd/server/nbfs/biz/core/file" - "github.com/nebulaim/telegramd/server/nbfs/biz/core/photo" - "math/rand" - "time" -) - -type PhotoServiceImpl struct { -} - -// rpc -// rpc nbfs_uploadPhotoFile(UploadPhotoFileRequest) returns (PhotoDataRsp); -func (s *PhotoServiceImpl) NbfsUploadPhotoFile(ctx context.Context, request *mtproto.UploadPhotoFileRequest) (*mtproto.PhotoDataRsp, error) { - glog.Infof("nbfs.uploadPhotoFile - request: %s", logger.JsonDebugData(request)) - if request.GetFile() == nil { - return nil, fmt.Errorf("bad request") - } - - inputFile := request.GetFile().GetData2() - - var ( - reply *mtproto.PhotoDataRsp - // isBigFile = request.GetFile().GetConstructor() == mtproto.TLConstructor_CRC32_inputFileBig - ) - - //// TODO(@benqi): 出错以后，回滚数据库操作 - //filePart, err := file.MakeFilePartData(request.OwnerId, inputFile.Id, false, isBigFile) - //if err != nil { - // glog.Error(err) - // return nil, err - //} - // - //var filename = core.NBFS_DATA_PATH + filePart.FilePath - //md5Checksum, _ := core.CalcMd5File(filename) - //if md5Checksum != inputFile.Md5Checksum { - // return nil, fmt.Errorf("check md5 error") - //} - - filePart, err := file.DoSavedFilePart(request.OwnerId, inputFile.Id) - if filePart.SavedMd5Hash != inputFile.Md5Checksum { - return nil, fmt.Errorf("check md5 error: %s, %s", filePart.SavedMd5Hash, inputFile.Md5Checksum) - } - - fileData, err := file.NewFileData(filePart.FilePartId, - filePart.SavedFilePath, - inputFile.Name, - filePart.FileSize, - filePart.SavedMd5Hash) - if err != nil { - glog.Error(err) - return nil, err - } - - photoId := base.NextSnowflakeId() - accessHash := rand.Int63() - szList, err := photo.UploadPhotoFile(photoId, accessHash, fileData.FilePath, fileData.Ext, false) - if err != nil { - glog.Error(err) - return nil, err - } - - reply = &mtproto.PhotoDataRsp{ - PhotoId: photoId, - AccessHash: accessHash, - Date: int32(time.Now().Unix()), - SizeList: szList, - } - - glog.Infof("nbfs.uploadPhotoFile - reply: %s", logger.JsonDebugData(reply)) - return reply, nil -} - -// rpc nbfs_getPhotoFileData(GetPhotoFileDataRequest) returns (PhotoDataRsp); -func (s *PhotoServiceImpl) NbfsGetPhotoFileData(ctx context.Context, request *mtproto.GetPhotoFileDataRequest) (*mtproto.PhotoDataRsp, error) { - glog.Infof("nbfs.getPhotoFileData - request: %s", logger.JsonDebugData(request)) - - var photoId = request.GetPhotoId() - szList := photo.GetPhotoSizeList(photoId) - reply := &mtproto.PhotoDataRsp{ - PhotoId: photoId, - SizeList: szList, - } - - glog.Infof("nbfs.getPhotoFileData - reply: %s", logger.JsonDebugData(reply)) - return reply, nil -} - -// inputMediaUploadedPhoto#2f37e231 flags:# file:InputFile caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; -func (s *PhotoServiceImpl) NbfsUploadedPhotoMedia(ctx context.Context, request *mtproto.NbfsUploadedPhotoMedia) (*mtproto.TLMessageMediaPhoto, error) { - glog.Infof("nbfs.uploadedPhotoMedia - request: %s", logger.JsonDebugData(request)) - - inputFile := request.GetMedia().GetFile().GetData2() - - var ( - // reply *mtproto.PhotoDataRsp - // isBigFile = request.GetMedia().GetFile().GetConstructor() == mtproto.TLConstructor_CRC32_inputFileBig - ) - - // TODO(@benqi): 出错以后，回滚数据库操作 - //filePart, err := file.MakeFilePartData(request.OwnerId, inputFile.Id, false, isBigFile) - //if err != nil { - // glog.Error(err) - // return nil, err - //} - // - //var filename = core.NBFS_DATA_PATH + filePart.FilePath - //md5Checksum, _ := core.CalcMd5File(filename) - //if md5Checksum != inputFile.Md5Checksum { - // return nil, fmt.Errorf("check md5 error") - //} - - filePart, err := file.DoSavedFilePart(request.OwnerId, inputFile.Id) - if filePart.SavedMd5Hash != inputFile.Md5Checksum { - return nil, fmt.Errorf("check md5 error: %s, %s", filePart.SavedMd5Hash, inputFile.Md5Checksum) - } - - fileData, err := file.NewFileData(filePart.FilePartId, - filePart.SavedMd5Hash, - inputFile.Name, - filePart.FileSize, - filePart.SavedMd5Hash) - if err != nil { - glog.Error(err) - return nil, err - } - - photoId := base.NextSnowflakeId() - accessHash := rand.Int63() - szList, err := photo.UploadPhotoFile(photoId, accessHash, fileData.FilePath, fileData.Ext, false) - if err != nil { - glog.Error(err) - return nil, err - } - - photo := &mtproto.TLPhoto{Data2: &mtproto.Photo_Data{ - Id: photoId, - HasStickers: false, - AccessHash: accessHash, - Date: int32(time.Now().Unix()), - Sizes: szList, - }} - - // photo:flags.0?Photo caption:flags.1?string ttl_seconds:flags.2?int - var reply = &mtproto.TLMessageMediaPhoto{Data2: &mtproto.MessageMedia_Data{ - Photo_1: photo.To_Photo(), - Caption: request.GetMedia().GetCaption(), - TtlSeconds: request.GetMedia().GetTtlSeconds(), - }} - - glog.Infof("nbfs.uploadedPhotoMedia - reply: %s", logger.JsonDebugData(reply)) - return nil, nil -} - -// inputMediaUploadedDocument#e39621fd flags:# file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; -func (s *PhotoServiceImpl) NbfsUploadedDocumentMedia(ctx context.Context, request *mtproto.NbfsUploadedDocumentMedia) (*mtproto.TLMessageMediaDocument, error) { - glog.Infof("nbfs.uploadedDocumentMedia - request: %s", logger.JsonDebugData(request)) - - var ( - inputFile = request.GetMedia().GetFile().GetData2() - // inputThumb = request.GetMedia().GetThumb() - // reply *mtproto.PhotoDataRsp - // isBigFile = request.GetMedia().GetFile().GetConstructor() == mtproto.TLConstructor_CRC32_inputFileBig - thumb *mtproto.PhotoSize - ) - - // TODO(@benqi): 出错以后，回滚数据库操作 - //filePart, err := file.MakeFilePartData(request.OwnerId, inputFile.Id, false, isBigFile) - //if err != nil { - // glog.Error(err) - // return nil, err - //} - - // var filename = core.NBFS_DATA_PATH + filePart.FilePath - filePart, err := file.DoSavedFilePart(request.OwnerId, inputFile.Id) - // core.CalcMd5File(filename) - if filePart.SavedMd5Hash != inputFile.Md5Checksum { - return nil, fmt.Errorf("check md5 error: %s, %s", filePart.SavedMd5Hash, inputFile.Md5Checksum) - } - - fileData, err := file.NewFileData(filePart.FilePartId, - filePart.SavedFilePath, - inputFile.Name, - filePart.FileSize, - filePart.SavedMd5Hash) - if err != nil { - glog.Error(err) - return nil, err - } - - // upload document file - documentId := base.NextSnowflakeId() - data, _ := document2.DoUploadedDocumentFile(documentId, - fileData.FilePath, - int32(fileData.FileSize), - fileData.UploadName, - fileData.Ext, - request.GetMedia().GetMimeType(), - 0) - - //thumb := &mtproto.TLPhotoSizeEmpty{Data2: &mtproto.PhotoSize_Data{ - // Type: "", - //}} - if request.GetMedia().GetThumb() != nil { - thumbFile := request.GetMedia().GetThumb().GetData2() - // var filename = core.NBFS_DATA_PATH + filePart.FilePath - filePart, err := file.DoSavedFilePart(request.OwnerId, thumbFile.Id) - // core.CalcMd5File(filename) - if filePart.SavedMd5Hash != thumbFile.Md5Checksum { - return nil, fmt.Errorf("check md5 error: %s, %s", filePart.SavedMd5Hash, thumbFile.Md5Checksum) - } - - fileData, err := file.NewFileData(filePart.FilePartId, - filePart.SavedFilePath, - thumbFile.Name, - filePart.FileSize, - filePart.SavedMd5Hash) - if err != nil { - glog.Error(err) - return nil, err - } - - photoId := base.NextSnowflakeId() - accessHash := rand.Int63() - szList, err := photo.UploadPhotoFile(photoId, accessHash, fileData.FilePath, fileData.Ext, false) - if err != nil { - glog.Error(err) - return nil, err - } - - thumb = &mtproto.PhotoSize{ - Constructor: mtproto.TLConstructor_CRC32_photoSize, - Data2: szList[0].GetData2(), - } - if thumb.Data2.Size == 0 { - thumb.Data2.Size = int32(len(thumb.Data2.Bytes)) - } - } else { - thumb = &mtproto.PhotoSize{ - Constructor: mtproto.TLConstructor_CRC32_photoSizeEmpty, - Data2: &mtproto.PhotoSize_Data{ - Type: "s", - }, - } - } - - document := &mtproto.TLDocument{Data2: &mtproto.Document_Data{ - Id: documentId, - AccessHash: data.AccessHash, - Date: int32(time.Now().Unix()), - MimeType: data.MimeType, - Size: int32(data.FileSize), - Thumb: thumb, - DcId: 2, - Version: 0, - Attributes: request.GetMedia().GetAttributes(), - }} - - // messageMediaDocument#7c4414d3 flags:# document:flags.0?Document caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; - var reply = &mtproto.TLMessageMediaDocument{Data2: &mtproto.MessageMedia_Data{ - Document: document.To_Document(), - Caption: request.GetMedia().GetCaption(), - TtlSeconds: request.GetMedia().GetTtlSeconds(), - }} - - glog.Infof("nbfs.uploadedDocumentMedia - reply: %s", logger.JsonDebugData(reply)) - return reply, nil -} - -// rpc nbfs_getDocument(DocumentId) returns (PhotoDataRsp); -func (s *PhotoServiceImpl) NbfsGetDocument(ctx context.Context, request *mtproto.DocumentId) (*mtproto.Document, error) { - glog.Infof("nbfs_getDocument - request: %s", logger.JsonDebugData(request)) - - reply := document2.GetDocument(request.Id, request.AccessHash, request.Version) - - glog.Infof("nbfs_getDocument - reply: %s", logger.JsonDebugData(reply)) - return reply, nil -} - -// rpc nbfs_getDocumentList(DocumentIdList) returns (DocumentList); -func (s *PhotoServiceImpl) NbfsGetDocumentList(ctx context.Context, request *mtproto.DocumentIdList) (*mtproto.DocumentList, error) { - glog.Infof("nbfs_getDocumentList - request: %s", logger.JsonDebugData(request)) - - documents := document2.GetDocumentList(request.IdList) - - glog.Infof("nbfs_getDocumentList - reply: %s", logger.JsonDebugData(documents)) - return &mtproto.DocumentList{Documents: documents}, nil -} diff --git a/server/nbfs/upload/rpc/README.md b/server/nbfs/upload/rpc/README.md deleted file mode 100644 index dffbafa9..00000000 --- a/server/nbfs/upload/rpc/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# upload服务 - -telegram客户端实现里，upload和download是两个独立的连接，相应的会有upload和download独立的服务。 - -当前先简单实现upload和download逻辑 - -后续也将upload和download服务独立出来，并引入独立的图片存储服务和文件存储系统 diff --git a/server/upload/README.md b/server/upload/README.md new file mode 100644 index 00000000..e3c5344e --- /dev/null +++ b/server/upload/README.md @@ -0,0 +1,12 @@ +# upload服务 +> telegram客户端实现里，upload和download是两个独立的连接，相应的会有upload和download独立的服务。 +> +> 当前先简单实现upload和download逻辑 +> +> 后续也将upload和download服务独立出来，并引入独立的图片存储服务和文件存储系统 + +## TODO +未使用分布式文件系统 + +## 多机器部署注意事项 +upload多机部署时，需要使用nfs，将/opt/nbfs目录mount到本机 diff --git a/server/upload/server/conf.go b/server/upload/server/conf.go new file mode 100644 index 00000000..c43a812f --- /dev/null +++ b/server/upload/server/conf.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "flag" + "fmt" + "github.com/BurntSushi/toml" + "github.com/nebulaim/telegramd/baselib/grpc_util" +) + +var ( + confPath string + Conf *uploadConfig +) + +type uploadConfig struct { + ServerId int32 // 服务器ID + DataPath string + RpcServer *grpc_util.RPCServerConfig +} + +func (c *uploadConfig) String() string { + return fmt.Sprintf("{server_id: %d, server: %v}", c.ServerId, c.RpcServer) +} + +func init() { + flag.StringVar(&confPath, "conf", "./upload.toml", "config path") +} + +func InitializeConfig() (err error) { + _, err = toml.DecodeFile(confPath, &Conf) + if err != nil { + err = fmt.Errorf("decode file %s error: %v", confPath, err) + } + return +} diff --git a/server/nbfs/upload/rpc/upload.getCdnFileHashes_handler.go b/server/upload/server/upload.getCdnFileHashes_handler.go similarity index 98% rename from server/nbfs/upload/rpc/upload.getCdnFileHashes_handler.go rename to server/upload/server/upload.getCdnFileHashes_handler.go index 1550df0b..db472949 100644 --- a/server/nbfs/upload/rpc/upload.getCdnFileHashes_handler.go +++ b/server/upload/server/upload.getCdnFileHashes_handler.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package rpc +package server import ( "fmt" diff --git a/server/nbfs/upload/rpc/upload.getCdnFile_handler.go b/server/upload/server/upload.getCdnFile_handler.go similarity index 98% rename from server/nbfs/upload/rpc/upload.getCdnFile_handler.go rename to server/upload/server/upload.getCdnFile_handler.go index 9fa8db51..7d9e0dc0 100644 --- a/server/nbfs/upload/rpc/upload.getCdnFile_handler.go +++ b/server/upload/server/upload.getCdnFile_handler.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package rpc +package server import ( "fmt" diff --git a/server/nbfs/upload/rpc/upload.getFile_handler.go b/server/upload/server/upload.getFile_handler.go similarity index 52% rename from server/nbfs/upload/rpc/upload.getFile_handler.go rename to server/upload/server/upload.getFile_handler.go index 3f67aca8..eb62d7c7 100644 --- a/server/nbfs/upload/rpc/upload.getFile_handler.go +++ b/server/upload/server/upload.getFile_handler.go @@ -15,16 +15,13 @@ * limitations under the License. */ -package rpc +package server import ( - "fmt" "github.com/golang/glog" "github.com/nebulaim/telegramd/baselib/grpc_util" "github.com/nebulaim/telegramd/baselib/logger" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/core/document" - photo2 "github.com/nebulaim/telegramd/server/nbfs/biz/core/photo" "golang.org/x/net/context" ) @@ -32,40 +29,10 @@ import ( func (s *UploadServiceImpl) UploadGetFile(ctx context.Context, request *mtproto.TLUploadGetFile) (*mtproto.Upload_File, error) { md := grpc_util.RpcMetadataFromIncoming(ctx) glog.Infof("upload.getFile#e3a6cfb5 - metadata: %s, request: %s", logger.JsonDebugData(md), logger.JsonDebugData(request)) - var ( - uploadFile *mtproto.Upload_File - err error - ) - switch request.GetLocation().GetConstructor() { - case mtproto.TLConstructor_CRC32_inputFileLocation: - fileLocation := request.GetLocation().To_InputFileLocation() - uploadFile, err = photo2.GetPhotoFileData(fileLocation.GetVolumeId(), - fileLocation.GetLocalId(), - fileLocation.GetSecret(), - request.GetOffset(), - request.GetLimit()) - case mtproto.TLConstructor_CRC32_inputEncryptedFileLocation: - case mtproto.TLConstructor_CRC32_inputDocumentFileLocation: - fileLocation := request.GetLocation().To_InputDocumentFileLocation() - uploadFile, err = document.GetDocumentFileData(fileLocation.GetId(), - fileLocation.GetAccessHash(), - fileLocation.GetVersion(), - request.GetOffset(), - request.GetLimit()) - case mtproto.TLConstructor_CRC32_inputDocumentFileLocationLayer11: - fileLocation := request.GetLocation().To_InputDocumentFileLocation() - uploadFile, err = document.GetDocumentFileData(fileLocation.GetId(), - fileLocation.GetAccessHash(), - fileLocation.GetVersion(), - request.GetOffset(), - request.GetLimit()) - default: - err = fmt.Errorf("invalid InputFileLocation type: %d", request.GetLocation().GetConstructor()) - } + uploadFile, err := s.NbfsFacade.DownloadFile(request.GetLocation(), request.GetOffset(), request.GetLimit()) if err != nil { glog.Error(err) - return nil, err } // type:storage.FileType mtime:int bytes:bytes @@ -73,6 +40,5 @@ func (s *UploadServiceImpl) UploadGetFile(ctx context.Context, request *mtproto. uploadFile.GetData2().GetType(), uploadFile.GetData2().GetMtime(), len(uploadFile.GetData2().GetBytes())) - - return uploadFile, err + return uploadFile, nil } diff --git a/server/nbfs/upload/rpc/upload.getWebFile_handler.go b/server/upload/server/upload.getWebFile_handler.go similarity index 98% rename from server/nbfs/upload/rpc/upload.getWebFile_handler.go rename to server/upload/server/upload.getWebFile_handler.go index 23e174d4..f97583e5 100644 --- a/server/nbfs/upload/rpc/upload.getWebFile_handler.go +++ b/server/upload/server/upload.getWebFile_handler.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package rpc +package server import ( "fmt" diff --git a/server/nbfs/upload/rpc/upload.reuploadCdnFile_handler.go b/server/upload/server/upload.reuploadCdnFile_handler.go similarity index 98% rename from server/nbfs/upload/rpc/upload.reuploadCdnFile_handler.go rename to server/upload/server/upload.reuploadCdnFile_handler.go index 6cbd719a..55431517 100644 --- a/server/nbfs/upload/rpc/upload.reuploadCdnFile_handler.go +++ b/server/upload/server/upload.reuploadCdnFile_handler.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package rpc +package server import ( "fmt" diff --git a/server/nbfs/upload/rpc/upload.saveBigFilePart_handler.go b/server/upload/server/upload.saveBigFilePart_handler.go similarity index 77% rename from server/nbfs/upload/rpc/upload.saveBigFilePart_handler.go rename to server/upload/server/upload.saveBigFilePart_handler.go index b56a4598..45408d03 100644 --- a/server/nbfs/upload/rpc/upload.saveBigFilePart_handler.go +++ b/server/upload/server/upload.saveBigFilePart_handler.go @@ -15,15 +15,15 @@ * limitations under the License. */ -package rpc +package server import ( "github.com/golang/glog" "github.com/nebulaim/telegramd/baselib/grpc_util" "github.com/nebulaim/telegramd/baselib/logger" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/core/file" "golang.org/x/net/context" + "github.com/nebulaim/telegramd/service/nbfs/cachefs" ) // upload.saveBigFilePart#de7b673d file_id:long file_part:int file_total_parts:int bytes:bytes = Bool; @@ -35,15 +35,8 @@ func (s *UploadServiceImpl) UploadSaveBigFilePart(ctx context.Context, request * request.FilePart, len(request.Bytes)) - // TODO(@benqi): Check file_total_parts >= bigFileSize/kMaxFileSize, Check len(bytes) <= kMaxFilePartSize, Check file_part <= file_total_parts - - filePartLogic, err := file.MakeFilePartData(md.AuthId, request.FileId, request.FilePart == 0, true) - if err != nil { - glog.Error(err) - return nil, err - } - - err = filePartLogic.SaveFilePart(request.FilePart, request.Bytes) + f := cachefs.NewCacheFile(md.AuthId, request.FileId) + err := f.WriteFilePartData(request.FilePart, request.Bytes) if err != nil { glog.Error(err) return nil, err diff --git a/server/nbfs/upload/rpc/upload.saveFilePart_handler.go b/server/upload/server/upload.saveFilePart_handler.go similarity index 78% rename from server/nbfs/upload/rpc/upload.saveFilePart_handler.go rename to server/upload/server/upload.saveFilePart_handler.go index f0c78331..e45bdb22 100644 --- a/server/nbfs/upload/rpc/upload.saveFilePart_handler.go +++ b/server/upload/server/upload.saveFilePart_handler.go @@ -15,15 +15,15 @@ * limitations under the License. */ -package rpc +package server import ( "github.com/golang/glog" "github.com/nebulaim/telegramd/baselib/grpc_util" "github.com/nebulaim/telegramd/baselib/logger" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/core/file" "golang.org/x/net/context" + "github.com/nebulaim/telegramd/service/nbfs/cachefs" ) // upload.saveFilePart#b304a621 file_id:long file_part:int bytes:bytes = Bool; @@ -35,15 +35,8 @@ func (s *UploadServiceImpl) UploadSaveFilePart(ctx context.Context, request *mtp request.FilePart, len(request.Bytes)) - // TODO(@benqi): Check file_part <= bigFileSize/kMaxFileSize, Check len(bytes) <= kMaxFilePartSize - - filePartLogic, err := file.MakeFilePartData(md.AuthId, request.FileId, request.FilePart == 0, false) - if err != nil { - glog.Error(err) - return nil, err - } - - err = filePartLogic.SaveFilePart(request.FilePart, request.Bytes) + f := cachefs.NewCacheFile(md.AuthId, request.FileId) + err := f.WriteFilePartData(request.FilePart, request.Bytes) if err != nil { glog.Error(err) return nil, err diff --git a/server/upload/server/upload_server.go b/server/upload/server/upload_server.go new file mode 100644 index 00000000..8c634d23 --- /dev/null +++ b/server/upload/server/upload_server.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "github.com/golang/glog" + "github.com/nebulaim/telegramd/baselib/grpc_util" + "google.golang.org/grpc" + "github.com/nebulaim/telegramd/proto/mtproto" +) + +type uploadServer struct { + rpcServer *grpc_util.RPCServer +} + +func NewUploadServer() *uploadServer { + return &uploadServer{} +} + +// AppInstance interface +func (s *uploadServer) Initialize() error { + glog.Infof("uploadServer - initialize...") + + err := InitializeConfig() + if err != nil { + glog.Fatal(err) + return err + } + glog.Info("uploadServer - load conf: ", Conf) + + // cachefs.InitCacheFS(Conf.DataPath) + // 初始化mysql_client、redis_client + // redis_client.InstallRedisClientManager(Conf.Redis) + // mysql_client.InstallMysqlClientManager(Conf.Mysql) + s.rpcServer = grpc_util.NewRpcServer(Conf.RpcServer.Addr, &Conf.RpcServer.RpcDiscovery) + return nil +} + +func (s *uploadServer) RunLoop() { + glog.Infof("uploadServer - runLoop...") + + // TODO(@benqi): check error + s.rpcServer.Serve(func(s2 *grpc.Server) { + impl := NewUploadServiceImpl(Conf.ServerId, Conf.DataPath) + mtproto.RegisterRPCUploadServer(s2, impl) + }) +} + +func (s *uploadServer) Destroy() { + glog.Infof("uploadServer - destroy...") + s.rpcServer.Stop() + //time.Sleep(1*time.Second) +} diff --git a/server/upload/server/upload_service_impl.go b/server/upload/server/upload_service_impl.go new file mode 100644 index 00000000..86c04894 --- /dev/null +++ b/server/upload/server/upload_service_impl.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "github.com/nebulaim/telegramd/service/nbfs/nbfs" + "github.com/nebulaim/telegramd/baselib/base" + "github.com/nebulaim/telegramd/service/nbfs/cachefs" +) + +type UploadServiceImpl struct { + nbfs_client.NbfsFacade +} + +func NewUploadServiceImpl(serverId int32, dataPath string) *UploadServiceImpl { + if dataPath == "" { + dataPath = "/opt/nbfs" + } + + cachefs.InitCacheFS(dataPath) + + s := &UploadServiceImpl{} + s.NbfsFacade, _ = nbfs_client.NewNbfsFacade("local", base.Int32ToString(serverId)) + return s +} diff --git a/server/nbfs/biz/base/id.go b/server/upload/upload.go similarity index 66% rename from server/nbfs/biz/base/id.go rename to server/upload/upload.go index 5a8e2ed8..9f9122d6 100644 --- a/server/nbfs/biz/base/id.go +++ b/server/upload/upload.go @@ -15,31 +15,23 @@ * limitations under the License. */ -package base +package main import ( "flag" - "github.com/nebulaim/telegramd/baselib/snowflake" -) - -var id *snowflake.IdWorker - -// = &snowflake.IdWorker{ -// -//} - -const ( - workerId = int64(1) - dataCenterId = int64(1) - twepoch = int64(1288834974657) + "github.com/nebulaim/telegramd/baselib/app" + _ "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/server/upload/server" ) func init() { - flag.Parse() - id, _ = snowflake.NewIdWorker(workerId, dataCenterId, twepoch) + flag.Set("alsologtostderr", "true") + flag.Set("log_dir", "false") } -func NextSnowflakeId() int64 { - r, _ := id.NextId() - return r +func main() { + flag.Parse() + + instance := server.NewUploadServer() + app.DoMainAppInstance(instance) } diff --git a/server/upload/upload.toml b/server/upload/upload.toml new file mode 100644 index 00000000..521efb48 --- /dev/null +++ b/server/upload/upload.toml @@ -0,0 +1,18 @@ +# biz_server.toml + +ver = "0.0.1" +#logPath = "/tmp/biz_server.log" + +[nbfs] +dataPath = "/opt/nbfs" + +[rpcServer] +addr = "0.0.0.0:10004" + +[rpcServer.rpcDiscovery] +serviceName = "upload" +nodeID = "node1" +rPCAddr = "127.0.0.1:10004" +etcdAddrs = ["http://127.0.0.1:2379"] +interval = "2s" +tTL = "10s" diff --git a/service/document/README.md b/service/document/README.md new file mode 100644 index 00000000..03cab904 --- /dev/null +++ b/service/document/README.md @@ -0,0 +1,7 @@ +# document服务 + +## TODO +nbfs未使用分布式文件系统 + +## 多机器部署注意事项 +document多机部署时，需要使用nfs，将/opt/nbfs目录mount到本机 diff --git a/server/nbfs/upload/rpc/upload_service_impl.go b/service/document/biz/core/core.go similarity index 79% rename from server/nbfs/upload/rpc/upload_service_impl.go rename to service/document/biz/core/core.go index 2fe1dd31..a7891df6 100644 --- a/server/nbfs/upload/rpc/upload_service_impl.go +++ b/service/document/biz/core/core.go @@ -15,8 +15,12 @@ * limitations under the License. */ -package rpc +package core -type UploadServiceImpl struct { - DataPath string +import ( + "github.com/nebulaim/telegramd/proto/mtproto" +) + +type PhotoCallback interface { + GetPhotoSizeList(photoId int64) (sizes []*mtproto.PhotoSize) } diff --git a/service/document/biz/core/document/document_file.go b/service/document/biz/core/document/document_file.go new file mode 100644 index 00000000..2ced871d --- /dev/null +++ b/service/document/biz/core/document/document_file.go @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package document + +import ( + "encoding/json" + "github.com/golang/glog" + "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/service/document/biz/dal/dataobject" + "time" + "github.com/nebulaim/telegramd/service/nbfs/proto" +) + +type documentData struct { + *dataobject.DocumentsDO +} + +func (m *DocumentModel) DoUploadedDocumentFile2(fileMD *nbfs.DocumentFileMetadata, thumbId int64) (*documentData, error) { + data := &dataobject.DocumentsDO{ + DocumentId: fileMD.DocumentId, + AccessHash: fileMD.AccessHash, + DcId: fileMD.DcId, + FilePath: fileMD.FilePath, + FileSize: fileMD.FileSize, + UploadedFileName: fileMD.UploadedFileName, + Ext: fileMD.Ext, + MimeType: fileMD.MimeType, + ThumbId: thumbId, + Version: 0, + } + data.Id = m.dao.DocumentsDAO.Insert(data) + return &documentData{DocumentsDO: data}, nil +} + +func (m *DocumentModel) makeDocumentByDO(do *dataobject.DocumentsDO) *mtproto.Document { + var ( + thumb *mtproto.PhotoSize + document *mtproto.Document + ) + + if do == nil { + document = mtproto.NewTLDocumentEmpty().To_Document() + } else { + if do.ThumbId != 0 { + sizeList := m.cb.GetPhotoSizeList(do.ThumbId) + if len(sizeList) > 0 { + thumb = sizeList[0] + } + } + if thumb == nil { + thumb = mtproto.NewTLPhotoSizeEmpty().To_PhotoSize() + } + + attributes := &mtproto.DocumentAttributeList{} + err := json.Unmarshal([]byte(do.Attributes), attributes) + if err != nil { + glog.Error(err) + attributes.Attributes = []*mtproto.DocumentAttribute{} + } + + // if do.Attributes + document = &mtproto.Document{ + Constructor: mtproto.TLConstructor_CRC32_document, + Data2: &mtproto.Document_Data{ + Id: do.DocumentId, + AccessHash: do.AccessHash, + Date: int32(time.Now().Unix()), + MimeType: do.MimeType, + Size: do.FileSize, + Thumb: thumb, + DcId: 2, + Version: do.Version, + Attributes: attributes.Attributes, + }, + } + } + + return document +} + +func (m *DocumentModel) GetDocument(id, accessHash int64, version int32) *mtproto.Document { + do := m.dao.DocumentsDAO.SelectByFileLocation(id, accessHash, version) + if do == nil { + glog.Warning("") + } + return m.makeDocumentByDO(do) +} + +func (m *DocumentModel) GetDocumentList(idList []int64) []*mtproto.Document { + doList := m.dao.DocumentsDAO.SelectByIdList(idList) + documetList := make([]*mtproto.Document, len(doList)) + for i := 0; i < len(doList); i++ { + documetList[i] = m.makeDocumentByDO(&doList[i]) + } + return documetList +} diff --git a/server/nbfs/biz/core/document/document_file_test.go b/service/document/biz/core/document/document_file_test.go similarity index 100% rename from server/nbfs/biz/core/document/document_file_test.go rename to service/document/biz/core/document/document_file_test.go diff --git a/service/document/biz/core/document/model.go b/service/document/biz/core/document/model.go new file mode 100644 index 00000000..82602d7b --- /dev/null +++ b/service/document/biz/core/document/model.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package document + +import ( + "github.com/nebulaim/telegramd/service/idgen/client" + "github.com/nebulaim/telegramd/baselib/mysql_client" + "github.com/golang/glog" + "github.com/nebulaim/telegramd/baselib/base" + "github.com/nebulaim/telegramd/service/document/biz/dal/dao/mysql_dao" + "github.com/nebulaim/telegramd/service/document/biz/core" +) + +type documentsDAO struct { + *mysql_dao.DocumentsDAO + // *mysql_dao.FilePartsDAO + idgen.UUIDGen + //idgen.SeqIDGen +} + +type DocumentModel struct { + // nbfsDataPath string + dao *documentsDAO + cb core.PhotoCallback +} + +func NewDocumentModel(serverId int32, dbName, redisName string, cb core.PhotoCallback) *DocumentModel { + m := &DocumentModel{dao: &documentsDAO{}, cb: cb} + db := mysql_client.GetMysqlClient(dbName) + if db == nil { + glog.Fatal("not found db: ", dbName) + } + + m.dao.DocumentsDAO = mysql_dao.NewDocumentsDAO(db) + // m.dao.FilePartsDAO = mysql_dao.NewFilePartsDAO(db) + + var err error + m.dao.UUIDGen, err = idgen.NewUUIDGen("snowflake", base.Int32ToString(serverId)) + if err != nil { + glog.Fatal("uuidgen init error: ", err) + } + //m.dao.SeqIDGen, _ = idgen.NewSeqIDGen("redis", redisName) + //if err != nil { + // glog.Fatal("seqidgen init error: ", err) + //} + return m +} diff --git a/service/document/biz/core/photo/model.go b/service/document/biz/core/photo/model.go new file mode 100644 index 00000000..4906b0d5 --- /dev/null +++ b/service/document/biz/core/photo/model.go @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package photo + +import ( + "github.com/nebulaim/telegramd/service/idgen/client" + "github.com/nebulaim/telegramd/baselib/mysql_client" + "github.com/golang/glog" + "github.com/nebulaim/telegramd/baselib/base" + "github.com/nebulaim/telegramd/service/document/biz/dal/dao/mysql_dao" +) + +type photosDAO struct { + *mysql_dao.PhotoDatasDAO + // *mysql_dao.FilePartsDAO + idgen.UUIDGen + //idgen.SeqIDGen +} + +type PhotoModel struct { + dao *photosDAO +} + +func NewPhotoModel(serverId int32, dbName, redisName string) *PhotoModel { + m := &PhotoModel{dao: &photosDAO{}} + db := mysql_client.GetMysqlClient(dbName) + if db == nil { + glog.Fatal("not found db: ", dbName) + } + + m.dao.PhotoDatasDAO = mysql_dao.NewPhotoDatasDAO(db) + // m.dao.FilePartsDAO = mysql_dao.NewFilePartsDAO(db) + + var err error + m.dao.UUIDGen, err = idgen.NewUUIDGen("snowflake", base.Int32ToString(serverId)) + if err != nil { + glog.Fatal("uuidgen init error: ", err) + } + //m.dao.SeqIDGen, _ = idgen.NewSeqIDGen("redis", redisName) + //if err != nil { + // glog.Fatal("seqidgen init error: ", err) + //} + return m +} diff --git a/service/document/biz/core/photo/photo_file.go b/service/document/biz/core/photo/photo_file.go new file mode 100644 index 00000000..eb054bea --- /dev/null +++ b/service/document/biz/core/photo/photo_file.go @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package photo + +import ( + "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/service/nbfs/proto" + "github.com/nebulaim/telegramd/service/document/biz/dal/dataobject" +) + +const ( + kPhotoSizeOriginalType = "0" // client upload original photo + kPhotoSizeSmallType = "s" + kPhotoSizeMediumType = "m" + kPhotoSizeXLargeType = "x" + kPhotoSizeYLargeType = "y" + kPhotoSizeAType = "a" + kPhotoSizeBType = "b" + kPhotoSizeCType = "c" + + kPhotoSizeOriginalSize = 0 // client upload original photo + kPhotoSizeSmallSize = 90 + kPhotoSizeMediumSize = 320 + kPhotoSizeXLargeSize = 800 + kPhotoSizeYLargeSize = 1280 + kPhotoSizeASize = 160 + kPhotoSizeBSize = 320 + kPhotoSizeCSize = 640 + + kPhotoSizeAIndex = 4 +) + +var sizeList = []int{ + kPhotoSizeOriginalSize, + kPhotoSizeSmallSize, + kPhotoSizeMediumSize, + kPhotoSizeXLargeSize, + kPhotoSizeYLargeSize, + kPhotoSizeASize, + kPhotoSizeBSize, + kPhotoSizeCSize, +} + +func getSizeType(idx int) string { + switch idx { + case 0: + return kPhotoSizeOriginalType + case 1: + return kPhotoSizeSmallType + case 2: + return kPhotoSizeMediumType + case 3: + return kPhotoSizeXLargeType + case 4: + return kPhotoSizeYLargeType + case 5: + return kPhotoSizeAType + case 6: + return kPhotoSizeBType + case 7: + return kPhotoSizeCType + } + + return "" +} + +//storage.fileUnknown#aa963b05 = storage.FileType; +//storage.filePartial#40bc6f52 = storage.FileType; +//storage.fileJpeg#7efe0e = storage.FileType; +//storage.fileGif#cae1aadf = storage.FileType; +//storage.filePng#a4f63c0 = storage.FileType; +//storage.filePdf#ae1e508d = storage.FileType; +//storage.fileMp3#528a0677 = storage.FileType; +//storage.fileMov#4b09ebbc = storage.FileType; +//storage.fileMp4#b3cea0e4 = storage.FileType; +//storage.fileWebp#1081464c = storage.FileType; + +//func checkIsABC(idx int) bool { +// if idx == +//} + +//import "github.com/golang/glog" +// + +type resizeInfo struct { + isWidth bool + size int +} + +//////////////////////////////////////////////////////////////////////////// +func (m *PhotoModel) GetPhotoSizeList(photoId int64) (sizes []*mtproto.PhotoSize) { + doList := m.dao.PhotoDatasDAO.SelectListByPhotoId(photoId) + sizes = make([]*mtproto.PhotoSize, 0, len(doList)) + for i := 1; i < len(doList); i++ { + sizeData := &mtproto.PhotoSize_Data{ + Type: getSizeType(int(doList[i].LocalId)), + W: doList[i].Width, + H: doList[i].Height, + Size: doList[i].FileSize, + Location: &mtproto.FileLocation{ + Constructor: mtproto.TLConstructor_CRC32_fileLocation, + Data2: &mtproto.FileLocation_Data{ + VolumeId: doList[i].VolumeId, + LocalId: int32(doList[i].LocalId), + Secret: doList[i].AccessHash, + DcId: doList[i].DcId, + }, + }, + } + + sizes = append(sizes, &mtproto.PhotoSize{ + Constructor: mtproto.TLConstructor_CRC32_photoSize, + Data2: sizeData, + }) + + // TODO(@benqi): add photoCachedSize + //if i == 1 { + // var filename = core.NBFS_DATA_PATH + doList[i].FilePath + // cacheData, err := ioutil.ReadFile(filename) + // if err != nil { + // glog.Errorf("read file %s error: %v", filename, err) + // sizeData.Bytes = []byte{} + // } else { + // sizeData.Bytes = cacheData + // } + // sizes = append(sizes, &mtproto.PhotoSize{ + // Constructor: mtproto.TLConstructor_CRC32_photoCachedSize, + // Data2: sizeData, + // }) + //} else { + // sizes = append(sizes, &mtproto.PhotoSize{ + // Constructor: mtproto.TLConstructor_CRC32_photoSize, + // Data2: sizeData, + // }) + //} + } + return +} + +func (m *PhotoModel) UploadPhotoFile2(fileMDList []*nbfs.PhotoFileMetadata) (photoId, accessHsh int64, sizeList []*mtproto.PhotoSize, err error) { + sizeList = make([]*mtproto.PhotoSize, 0, 4) + + for i, fileMD := range fileMDList { + photoDatasDO := &dataobject.PhotoDatasDO{ + PhotoId: fileMD.PhotoId, + PhotoType: fileMD.PhotoType, + DcId: fileMD.DcId, + VolumeId: fileMD.VolumeId, + LocalId: fileMD.LocalId, + AccessHash: fileMD.SecretId, + Width: fileMD.Width, + Height: fileMD.Height, + FileSize: fileMD.FileSize, + FilePath: fileMD.FilePath, + Ext: fileMD.Ext, + } + + // photoDatasDO.Bytes = imgBuf.Bytes() + m.dao.PhotoDatasDAO.Insert(photoDatasDO) + + photoSizeData := &mtproto.PhotoSize_Data{ + Type: getSizeType(int(fileMD.LocalId)), + W: photoDatasDO.Width, + H: photoDatasDO.Height, + // Size: int32(len(photoDatasDO.Bytes)), + Size: photoDatasDO.FileSize, + Location: &mtproto.FileLocation{ + Constructor: mtproto.TLConstructor_CRC32_fileLocation, + Data2: &mtproto.FileLocation_Data{ + VolumeId: photoDatasDO.VolumeId, + LocalId: fileMD.LocalId, + Secret: photoDatasDO.AccessHash, + DcId: photoDatasDO.DcId, + }, + }, + } + + if i == 0 { + photoId = fileMD.PhotoId + accessHsh = fileMD.SecretId + continue + } else { + size := &mtproto.PhotoSize{ + Constructor: mtproto.TLConstructor_CRC32_photoSize, + Data2: photoSizeData, + } + sizeList = append(sizeList, size) + } + } + return +} diff --git a/server/nbfs/biz/core/photo/photo_file_test.go b/service/document/biz/core/photo/photo_file_test.go similarity index 91% rename from server/nbfs/biz/core/photo/photo_file_test.go rename to service/document/biz/core/photo/photo_file_test.go index 0534e32e..fa6ac8af 100644 --- a/server/nbfs/biz/core/photo/photo_file_test.go +++ b/service/document/biz/core/photo/photo_file_test.go @@ -62,8 +62,9 @@ func TestUploadPhotoFile(t *testing.T) { return } - UploadPhotoFile(fileData.FileId, 1, fileData.FilePath, fileData.Ext, false) - UploadPhotoFile(fileData.FileId, 2, fileData.FilePath, fileData.Ext, true) + _ = fileData + // UploadPhotoFile(fileData.FileId, 1, fileData.FilePath, fileData.Ext, false) + // UploadPhotoFile(fileData.FileId, 2, fileData.FilePath, fileData.Ext, true) //func MakeFileDataByLoad(fileId, accessHash int64) (*fileData, error) { // diff --git a/server/nbfs/biz/dal/README.md b/service/document/biz/dal/README.md similarity index 100% rename from server/nbfs/biz/dal/README.md rename to service/document/biz/dal/README.md diff --git a/server/nbfs/biz/dal/dalgen/dalgen.sh b/service/document/biz/dal/dalgen/dalgen.sh similarity index 100% rename from server/nbfs/biz/dal/dalgen/dalgen.sh rename to service/document/biz/dal/dalgen/dalgen.sh diff --git a/server/nbfs/biz/dal/dalgen/dalgen.toml b/service/document/biz/dal/dalgen/dalgen.toml similarity index 100% rename from server/nbfs/biz/dal/dalgen/dalgen.toml rename to service/document/biz/dal/dalgen/dalgen.toml diff --git a/server/nbfs/biz/dal/dalgen/dalgen_all.sh b/service/document/biz/dal/dalgen/dalgen_all.sh similarity index 100% rename from server/nbfs/biz/dal/dalgen/dalgen_all.sh rename to service/document/biz/dal/dalgen/dalgen_all.sh diff --git a/server/nbfs/biz/dal/dalgen/nebula-dal-generator b/service/document/biz/dal/dalgen/nebula-dal-generator similarity index 100% rename from server/nbfs/biz/dal/dalgen/nebula-dal-generator rename to service/document/biz/dal/dalgen/nebula-dal-generator diff --git a/server/nbfs/biz/dal/dalgen/tpl/gen_dao.tpl b/service/document/biz/dal/dalgen/tpl/gen_dao.tpl similarity index 98% rename from server/nbfs/biz/dal/dalgen/tpl/gen_dao.tpl rename to service/document/biz/dal/dalgen/tpl/gen_dao.tpl index bd4dd68a..2a3f80cc 100644 --- a/server/nbfs/biz/dal/dalgen/tpl/gen_dao.tpl +++ b/service/document/biz/dal/dalgen/tpl/gen_dao.tpl @@ -18,8 +18,8 @@ package mysql_dao import( - "github.com/nebulaim/telegramd/mtproto" - "github.com/nebulaim/telegramd/nbfs/biz/dal/dataobject" + "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/service/document/biz/dal/dataobject" "github.com/jmoiron/sqlx" "github.com/golang/glog" "fmt" diff --git a/server/nbfs/biz/dal/dalgen/tpl/gen_do.tpl b/service/document/biz/dal/dalgen/tpl/gen_do.tpl similarity index 100% rename from server/nbfs/biz/dal/dalgen/tpl/gen_do.tpl rename to service/document/biz/dal/dalgen/tpl/gen_do.tpl diff --git a/server/nbfs/biz/dal/dao/dao_manager.go b/service/document/biz/dal/dao/dao_manager.go similarity index 100% rename from server/nbfs/biz/dal/dao/dao_manager.go rename to service/document/biz/dal/dao/dao_manager.go diff --git a/server/nbfs/biz/dal/dao/mysql_dao/documents_dao.go b/service/document/biz/dal/dao/mysql_dao/documents_dao.go similarity index 98% rename from server/nbfs/biz/dal/dao/mysql_dao/documents_dao.go rename to service/document/biz/dal/dao/mysql_dao/documents_dao.go index 391c4cfe..888d5692 100644 --- a/server/nbfs/biz/dal/dao/mysql_dao/documents_dao.go +++ b/service/document/biz/dal/dao/mysql_dao/documents_dao.go @@ -22,7 +22,7 @@ import ( "github.com/golang/glog" "github.com/jmoiron/sqlx" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" + "github.com/nebulaim/telegramd/service/document/biz/dal/dataobject" ) type DocumentsDAO struct { diff --git a/server/nbfs/biz/dal/dao/mysql_dao/gofmt.sh b/service/document/biz/dal/dao/mysql_dao/gofmt.sh similarity index 100% rename from server/nbfs/biz/dal/dao/mysql_dao/gofmt.sh rename to service/document/biz/dal/dao/mysql_dao/gofmt.sh diff --git a/server/nbfs/biz/dal/dao/mysql_dao/photo_datas_dao.go b/service/document/biz/dal/dao/mysql_dao/photo_datas_dao.go similarity index 98% rename from server/nbfs/biz/dal/dao/mysql_dao/photo_datas_dao.go rename to service/document/biz/dal/dao/mysql_dao/photo_datas_dao.go index 749ea8fc..d832596f 100644 --- a/server/nbfs/biz/dal/dao/mysql_dao/photo_datas_dao.go +++ b/service/document/biz/dal/dao/mysql_dao/photo_datas_dao.go @@ -22,7 +22,7 @@ import ( "github.com/golang/glog" "github.com/jmoiron/sqlx" "github.com/nebulaim/telegramd/proto/mtproto" - "github.com/nebulaim/telegramd/server/nbfs/biz/dal/dataobject" + "github.com/nebulaim/telegramd/service/document/biz/dal/dataobject" ) type PhotoDatasDAO struct { diff --git a/server/nbfs/biz/dal/dataobject/documents_do.go b/service/document/biz/dal/dataobject/documents_do.go similarity index 94% rename from server/nbfs/biz/dal/dataobject/documents_do.go rename to service/document/biz/dal/dataobject/documents_do.go index dd147a5a..a9d2117c 100644 --- a/server/nbfs/biz/dal/dataobject/documents_do.go +++ b/service/document/biz/dal/dataobject/documents_do.go @@ -30,6 +30,8 @@ type DocumentsDO struct { ThumbId int64 `db:"thumb_id"` Version int32 `db:"version"` Attributes string `db:"attributes"` + FileId int64 `db:"file_id"` + State int8 `db:"state"` CreatedAt string `db:"created_at"` UpdatedAt string `db:"updated_at"` } diff --git a/server/nbfs/biz/dal/dataobject/gofmt.sh b/service/document/biz/dal/dataobject/gofmt.sh similarity index 100% rename from server/nbfs/biz/dal/dataobject/gofmt.sh rename to service/document/biz/dal/dataobject/gofmt.sh diff --git a/server/nbfs/biz/dal/dataobject/photo_datas_do.go b/service/document/biz/dal/dataobject/photo_datas_do.go similarity index 94% rename from server/nbfs/biz/dal/dataobject/photo_datas_do.go rename to service/document/biz/dal/dataobject/photo_datas_do.go index 8a1d7251..29cba49f 100644 --- a/server/nbfs/biz/dal/dataobject/photo_datas_do.go +++ b/service/document/biz/dal/dataobject/photo_datas_do.go @@ -30,5 +30,7 @@ type PhotoDatasDO struct { FileSize int32 `db:"file_size"` FilePath string `db:"file_path"` Ext string `db:"ext"` + FileId int64 `db:"file_id"` + State int8 `db:"state"` CreatedAt string `db:"created_at"` } diff --git a/server/nbfs/biz/dal/tables/documents.xml b/service/document/biz/dal/tables/documents.xml similarity index 100% rename from server/nbfs/biz/dal/tables/documents.xml rename to service/document/biz/dal/tables/documents.xml diff --git a/server/nbfs/biz/dal/tables/photo_datas.xml b/service/document/biz/dal/tables/photo_datas.xml similarity index 100% rename from server/nbfs/biz/dal/tables/photo_datas.xml rename to service/document/biz/dal/tables/photo_datas.xml diff --git a/service/document/client/document.go b/service/document/client/document.go new file mode 100644 index 00000000..af53ee5d --- /dev/null +++ b/service/document/client/document.go @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package document_client + +import ( + "fmt" + "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/service/nbfs/proto" +) + +type DocumentFacade interface { + Initialize(config string) error + UploadPhotoFile(creatorId int64, file *mtproto.InputFile) ([]*nbfs.PhotoFileMetadata, error) + UploadProfilePhotoFile(creatorId int64, file *mtproto.InputFile) ([]*nbfs.PhotoFileMetadata, error) + UploadDocumentFile(creatorId int64, file *mtproto.InputFile) (*nbfs.DocumentFileMetadata, error) + // UploadFileParts(creatorId, filePartId int64) (bool, error) + DownloadFile(location *mtproto.InputFileLocation, offset, limit int32) (*mtproto.Upload_File, error) +} + +type Instance func() DocumentFacade + +var instances = make(map[string]Instance) + +func Register(name string, inst Instance) { + if inst == nil { + panic("register instance is nil") + } + if _, ok := instances[name]; ok { + panic("register called twice for instance " + name) + } + instances[name] = inst +} + +func NewDocumentFacade(name, config string) (inst DocumentFacade, err error) { + instanceFunc, ok := instances[name] + if !ok { + err = fmt.Errorf("unknown instance name %q (forgot to import?)", name) + return + } + inst = instanceFunc() + err = inst.Initialize(config) + if err != nil { + inst = nil + } + return +} diff --git a/server/nbfs/nbfs_client/nbfs_rpc_client.go b/service/document/client/rpc_document_client.go similarity index 97% rename from server/nbfs/nbfs_client/nbfs_rpc_client.go rename to service/document/client/rpc_document_client.go index 175928de..b2d625a4 100644 --- a/server/nbfs/nbfs_client/nbfs_rpc_client.go +++ b/service/document/client/rpc_document_client.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package nbfs_client +package document_client import ( "context" @@ -25,12 +25,12 @@ import ( "github.com/nebulaim/telegramd/proto/mtproto" ) -type nbfsClient struct { +type documentClient struct { client mtproto.RPCNbfsClient } var ( - nbfsInstance = &nbfsClient{} + nbfsInstance = &documentClient{} ) func InstallNbfsClient(discovery *service_discovery.ServiceDiscoveryClientConfig) { diff --git a/server/nbfs/photo/photo.go b/service/document/document.go similarity index 60% rename from server/nbfs/photo/photo.go rename to service/document/document.go index 0a7df4c2..0b201e58 100644 --- a/server/nbfs/photo/photo.go +++ b/service/document/document.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, https://github.com/nebulaim + * Copyright (c) 2017, https://github.com/nebulaim * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,5 +17,21 @@ package main +import ( + "flag" + "github.com/nebulaim/telegramd/baselib/app" + _ "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/service/document/service" +) + +func init() { + flag.Set("alsologtostderr", "true") + flag.Set("log_dir", "false") +} + func main() { + flag.Parse() + + instance := service.NewDocumentServer() + app.DoMainAppInstance(instance) } diff --git a/server/nbfs/nbfs.toml b/service/document/document.toml similarity index 67% rename from server/nbfs/nbfs.toml rename to service/document/document.toml index 5572ea6e..3b7425cc 100644 --- a/server/nbfs/nbfs.toml +++ b/service/document/document.toml @@ -6,17 +6,29 @@ ver = "0.0.1" [nbfs] dataPath = "/opt/nbfs" -[server] -addr = "0.0.0.0:10004" +[rpcServer] +addr = "0.0.0.0:10009" -[discovery] -serviceName = "nbfs" +[rpcServer.rpcDiscovery] +serviceName = "document" nodeID = "node1" -rPCAddr = "127.0.0.1:10004" +rPCAddr = "127.0.0.1:10009" etcdAddrs = ["http://127.0.0.1:2379"] interval = "2s" tTL = "10s" +[[redis]] +name = "cache" +addr = "127.0.0.1:6379" +idle = 100 +active = 100 +dialTimeout = "1s" +readTimeout = "1s" +writeTimeout = "1s" +idleTimeout = "10s" +dbNum = "0" +password = "" + [[mysql]] name = "immaster" dsn = "root:@/nebulaim?charset=utf8mb4" diff --git a/service/document/proto/build2.sh b/service/document/proto/build2.sh new file mode 100755 index 00000000..b7ddbbfa --- /dev/null +++ b/service/document/proto/build2.sh @@ -0,0 +1,7 @@ +#!/bin/sh +SRC_DIR=. +DST_DIR=. + +#./codegen.sh +#protoc -I=$SRC_DIR --go_out=$DST_DIR/ $SRC_DIR/*.proto +protoc -I=$SRC_DIR --go_out=plugins=grpc:$DST_DIR/ $SRC_DIR/*.proto diff --git a/server/nbfs/upload/upload.go b/service/document/proto/document.proto similarity index 68% rename from server/nbfs/upload/upload.go rename to service/document/proto/document.proto index 0a7df4c2..9771a522 100644 --- a/server/nbfs/upload/upload.go +++ b/service/document/proto/document.proto @@ -15,7 +15,13 @@ * limitations under the License. */ -package main +syntax = "proto3"; -func main() { -} +option java_multiple_files = true; +option java_package = "im.nebula.service.status"; +option java_outer_classname = "StatusProto"; +option optimize_for = CODE_SIZE; + +package document; + +//////////////////////////////////////////////////////////////////////////////////////// diff --git a/service/document/service/conf.go b/service/document/service/conf.go new file mode 100644 index 00000000..80f0a4b9 --- /dev/null +++ b/service/document/service/conf.go @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "flag" + "fmt" + "github.com/BurntSushi/toml" + "github.com/nebulaim/telegramd/baselib/grpc_util" + "github.com/nebulaim/telegramd/baselib/mysql_client" + "github.com/nebulaim/telegramd/baselib/redis_client" +) + +var ( + confPath string + Conf *documentConfig +) + +type documentConfig struct { + ServerId int32 // 服务器ID + DataPath string + Redis []redis_client.RedisConfig + Mysql []mysql_client.MySQLConfig + RpcServer *grpc_util.RPCServerConfig +} + +func (c *documentConfig) String() string { + return fmt.Sprintf("{server_id: %d, redis: %v. mysql: %v, server: %v}", + c.ServerId, + c.Redis, + c.Mysql, + c.RpcServer) +} + +func init() { + flag.StringVar(&confPath, "conf", "./document.toml", "config path") +} + +func InitializeConfig() (err error) { + _, err = toml.DecodeFile(confPath, &Conf) + if err != nil { + err = fmt.Errorf("decode file %s error: %v", confPath, err) + } + return +} diff --git a/service/document/service/document_server.go b/service/document/service/document_server.go new file mode 100644 index 00000000..3bcfc6a2 --- /dev/null +++ b/service/document/service/document_server.go @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "github.com/golang/glog" + "github.com/nebulaim/telegramd/baselib/grpc_util" + "github.com/nebulaim/telegramd/baselib/mysql_client" + "google.golang.org/grpc" + "github.com/nebulaim/telegramd/baselib/redis_client" + "github.com/nebulaim/telegramd/proto/mtproto" +) + +type documentServer struct { + rpcServer *grpc_util.RPCServer +} + +func NewDocumentServer() *documentServer { + return &documentServer{} +} + +// AppInstance interface +func (s *documentServer) Initialize() error { + glog.Infof("documentServer - initialize...") + + err := InitializeConfig() + if err != nil { + glog.Fatal(err) + return err + } + glog.Info("documentServer - load conf: ", Conf) + + // 初始化mysql_client、redis_client + redis_client.InstallRedisClientManager(Conf.Redis) + mysql_client.InstallMysqlClientManager(Conf.Mysql) + + s.rpcServer = grpc_util.NewRpcServer(Conf.RpcServer.Addr, &Conf.RpcServer.RpcDiscovery) + return nil +} + +func (s *documentServer) RunLoop() { + glog.Infof("documentServer - runLoop...") + + // TODO(@benqi): check error + s.rpcServer.Serve(func(s2 *grpc.Server) { + impl := NewDocumentServiceImpl(Conf.ServerId, Conf.DataPath, "immaster", "cache") + mtproto.RegisterRPCNbfsServer(s2, impl) + }) +} + +func (s *documentServer) Destroy() { + glog.Infof("documentServer - destroy...") + s.rpcServer.Stop() + //time.Sleep(1*time.Second) +} diff --git a/service/document/service/documtnt_service_impl.go b/service/document/service/documtnt_service_impl.go new file mode 100644 index 00000000..ddf37b6e --- /dev/null +++ b/service/document/service/documtnt_service_impl.go @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "context" + "fmt" + "github.com/golang/glog" + "github.com/nebulaim/telegramd/baselib/logger" + "github.com/nebulaim/telegramd/proto/mtproto" + document2 "github.com/nebulaim/telegramd/service/document/biz/core/document" + photo2 "github.com/nebulaim/telegramd/service/document/biz/core/photo" + "time" + "github.com/nebulaim/telegramd/service/nbfs/nbfs" + "github.com/nebulaim/telegramd/service/nbfs/cachefs" + "github.com/nebulaim/telegramd/baselib/base" +) + +type DocumentServiceImpl struct { + *document2.DocumentModel + *photo2.PhotoModel + nbfs_client.NbfsFacade +} + +func NewDocumentServiceImpl(serverId int32, dataPath string, dbName, redisName string) *DocumentServiceImpl { + if dataPath == "" { + dataPath = "/opt/nbfs" + } + + cachefs.InitCacheFS(dataPath) + + s := &DocumentServiceImpl{} + s.NbfsFacade, _ = nbfs_client.NewNbfsFacade("local", base.Int32ToString(serverId)) + s.PhotoModel = photo2.NewPhotoModel(serverId, dbName, redisName) + s.DocumentModel = document2.NewDocumentModel(serverId, dbName, redisName, s.PhotoModel) + + return s +} + +// rpc +// rpc nbfs_uploadPhotoFile(UploadPhotoFileRequest) returns (PhotoDataRsp); +func (s *DocumentServiceImpl) NbfsUploadPhotoFile(ctx context.Context, request *mtproto.UploadPhotoFileRequest) (*mtproto.PhotoDataRsp, error) { + glog.Infof("nbfs.uploadPhotoFile - request: %s", logger.JsonDebugData(request)) + + var ( + reply *mtproto.PhotoDataRsp + inputFile = request.GetFile() + ) + + if request.GetFile() == nil { + return nil, fmt.Errorf("bad request") + } + + fileMDList, err := s.NbfsFacade.UploadPhotoFile(request.OwnerId, inputFile) + if err != nil { + glog.Error(err) + return nil, err + } + + // glog.Info(fileMDList) + photoId, accessHash, szList, err := s.PhotoModel.UploadPhotoFile2(fileMDList) + if err != nil { + glog.Error(err) + return nil, err + } + + reply = &mtproto.PhotoDataRsp{ + PhotoId: photoId, + AccessHash: accessHash, + Date: int32(time.Now().Unix()), + SizeList: szList, + } + + // glog.Infof("nbfs.uploadPhotoFile - reply: %s", logger.JsonDebugData(reply)) + return reply, nil +} + +// rpc nbfs_getPhotoFileData(GetPhotoFileDataRequest) returns (PhotoDataRsp); +func (s *DocumentServiceImpl) NbfsGetPhotoFileData(ctx context.Context, request *mtproto.GetPhotoFileDataRequest) (*mtproto.PhotoDataRsp, error) { + glog.Infof("nbfs.getPhotoFileData - request: %s", logger.JsonDebugData(request)) + + var photoId = request.GetPhotoId() + szList := s.PhotoModel.GetPhotoSizeList(photoId) + reply := &mtproto.PhotoDataRsp{ + PhotoId: photoId, + SizeList: szList, + } + + // glog.Infof("nbfs.getPhotoFileData - reply: %s", logger.JsonDebugData(reply)) + return reply, nil +} + +// inputMediaUploadedPhoto#2f37e231 flags:# file:InputFile caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; +func (s *DocumentServiceImpl) NbfsUploadedPhotoMedia(ctx context.Context, request *mtproto.NbfsUploadedPhotoMedia) (*mtproto.TLMessageMediaPhoto, error) { + glog.Infof("nbfs.uploadedPhotoMedia - request: %s", logger.JsonDebugData(request)) + + + var ( + inputFile = request.GetMedia().GetFile() + ) + + fileMDList, err := s.NbfsFacade.UploadPhotoFile(request.OwnerId, inputFile) + if err != nil { + glog.Error(err) + return nil, err + } + + photoId, accessHash, szList, err := s.PhotoModel.UploadPhotoFile2(fileMDList) + if err != nil { + glog.Error(err) + return nil, err + } + + photo := &mtproto.TLPhoto{Data2: &mtproto.Photo_Data{ + Id: photoId, + HasStickers: false, + AccessHash: accessHash, + Date: int32(time.Now().Unix()), + Sizes: szList, + }} + + // photo:flags.0?Photo caption:flags.1?string ttl_seconds:flags.2?int + var reply = &mtproto.TLMessageMediaPhoto{Data2: &mtproto.MessageMedia_Data{ + Photo_1: photo.To_Photo(), + Caption: request.GetMedia().GetCaption(), + TtlSeconds: request.GetMedia().GetTtlSeconds(), + }} + + // glog.Info("nbfs.uploadedPhotoMedia - reply: %s", logger.JsonDebugData(reply)) + return reply, nil +} + +// inputMediaUploadedDocument#e39621fd flags:# file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; +func (s *DocumentServiceImpl) NbfsUploadedDocumentMedia(ctx context.Context, request *mtproto.NbfsUploadedDocumentMedia) (*mtproto.TLMessageMediaDocument, error) { + glog.Infof("nbfs.uploadedDocumentMedia - request: %s", logger.JsonDebugData(request)) + + var ( + inputFile = request.GetMedia().GetFile() + inputThumb = request.GetMedia().GetThumb() + media = request.GetMedia() + thumb *mtproto.PhotoSize + thumbId int64 + ) + + if media.GetThumb() != nil { + thumbFileMDList, err := s.NbfsFacade.UploadPhotoFile(request.OwnerId, inputThumb) + if err != nil { + glog.Error(err) + return nil, err + } + + var szList []*mtproto.PhotoSize + thumbId, _, szList, err = s.PhotoModel.UploadPhotoFile2(thumbFileMDList) + if err != nil { + glog.Error(err) + return nil, err + } + + thumb = &mtproto.PhotoSize{ + Constructor: mtproto.TLConstructor_CRC32_photoSize, + Data2: szList[0].GetData2(), + } + if thumb.Data2.Size == 0 { + thumb.Data2.Size = int32(len(thumb.Data2.Bytes)) + } + } else { + thumb = &mtproto.PhotoSize{ + Constructor: mtproto.TLConstructor_CRC32_photoSizeEmpty, + Data2: &mtproto.PhotoSize_Data{ + Type: "s", + }, + } + } + + fileMD, err := s.NbfsFacade.UploadDocumentFile(request.OwnerId, inputFile) + if err != nil { + glog.Error(err) + return nil, err + } + + fileMD.MimeType = request.GetMedia().GetMimeType() + data, err := s.DocumentModel.DoUploadedDocumentFile2(fileMD, thumbId) + if err != nil { + glog.Error(err) + return nil, err + } + + document := &mtproto.TLDocument{Data2: &mtproto.Document_Data{ + Id: data.DocumentId, + AccessHash: data.AccessHash, + Date: int32(time.Now().Unix()), + MimeType: data.MimeType, + Size: int32(data.FileSize), + Thumb: thumb, + DcId: fileMD.DcId, + Version: 0, + Attributes: media.GetAttributes(), + }} + + // messageMediaDocument#7c4414d3 flags:# document:flags.0?Document caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; + var reply = &mtproto.TLMessageMediaDocument{Data2: &mtproto.MessageMedia_Data{ + Document: document.To_Document(), + Caption: request.GetMedia().GetCaption(), + TtlSeconds: request.GetMedia().GetTtlSeconds(), + }} + + // glog.Infof("nbfs.uploadedDocumentMedia - reply: %s", logger.JsonDebugData(reply)) + return reply, nil +} + +// rpc nbfs_getDocument(DocumentId) returns (PhotoDataRsp); +func (s *DocumentServiceImpl) NbfsGetDocument(ctx context.Context, request *mtproto.DocumentId) (*mtproto.Document, error) { + glog.Infof("nbfs_getDocument - request: %s", logger.JsonDebugData(request)) + + reply := s.DocumentModel.GetDocument(request.Id, request.AccessHash, request.Version) + + // glog.Infof("nbfs_getDocument - reply: %s", logger.JsonDebugData(reply)) + return reply, nil +} + +// rpc nbfs_getDocumentList(DocumentIdList) returns (DocumentList); +func (s *DocumentServiceImpl) NbfsGetDocumentList(ctx context.Context, request *mtproto.DocumentIdList) (*mtproto.DocumentList, error) { + glog.Infof("nbfs_getDocumentList - request: %s", logger.JsonDebugData(request)) + + documents := s.DocumentModel.GetDocumentList(request.IdList) + + // glog.Infof("nbfs_getDocumentList - reply: %s", logger.JsonDebugData(documents)) + return &mtproto.DocumentList{Documents: documents}, nil +} diff --git a/service/nbfs/cachefs/cachefs.go b/service/nbfs/cachefs/cachefs.go new file mode 100644 index 00000000..a77efd15 --- /dev/null +++ b/service/nbfs/cachefs/cachefs.go @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cachefs + +import ( + "os" + "github.com/golang/glog" + "fmt" + "io/ioutil" + "crypto/md5" + "strings" + "path" +) + +//type CacheFileIndex struct { +// FileId int64 +// FileName string +// FileSize int64 +// FileTotalPart int32 +// FileParts []int32 +//} +// + +// serverId.creatorId.fileId.tmp + +var rootDataPath = "/opt/nbfs" +var subPaths = []string{"0", "a", "b", "c", "s", "m", "x", "y"} +// var uuidgen idgen.UUIDGen + +func InitCacheFS(dataPath string) error { + for _, p := range subPaths { + err := os.MkdirAll(rootDataPath + "/" + p, 0755) + if err != nil { + glog.Fatal("init cache fs error: ", err) + return err + } + } + return nil +} + +// 判断文件是否存在 +func pathExists(filePath string) (bool, error) { + _, err := os.Stat(filePath) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +//获取单个文件的大小 +func getFileSize(path string) int64 { + fileInfo, err := os.Stat(path) + if err != nil { + return 0 + } + return fileInfo.Size() +} + +func getFileExtName(filePath string) string { + var ext = path.Ext(filePath) + return strings.ToLower(ext) +} + +type cacheFile struct { + creatorId int64 + fileId int64 +} + +func NewCacheFile(creatorId, fileId int64) *cacheFile { + return &cacheFile{creatorId, fileId} +} + +func (f *cacheFile) WriteFilePartData(filePart int32, bytes []byte) error { + filePath := fmt.Sprintf("%s/0/%d.%d.parts", rootDataPath, f.creatorId, f.fileId) + + exist, err := pathExists(filePath) + if err != nil { + glog.Errorf("pathExists error![%v]", err) + return err + } + + if !exist { + err := os.Mkdir(filePath, 0755) + if err != nil { + glog.Errorf("mkdir failed![%v]\n", err) + return err + } + } + + fileName := fmt.Sprintf("%s/%d.part", filePath, filePart) + // exist, _ = pathExists(fileName) + // 直接覆盖 + err = ioutil.WriteFile(fileName, bytes, 0644) + if err != nil { + glog.Error(err) + return err + } + + return nil +} + +func (f *cacheFile) CheckFileParts(fileParts int32) bool { + for i := 0; i < int(fileParts); i++ { + filePath := fmt.Sprintf("%s/0/%d.%d.parts/%d.part", rootDataPath, f.creatorId, f.fileId, i) + filePartInfo, err := os.Stat(filePath) + if err != nil { + glog.Error(err) + return false + } + + if filePartInfo.IsDir() { + err = fmt.Errorf("exist dir - %s", filePath) + return false + } + } + return true +} + +func (f *cacheFile) Md5Checksum(fileParts int32) (string, error) { + md5Hash := md5.New() + for i := 0; i < int(fileParts); i++ { + filePath := fmt.Sprintf("%s/0/%d.%d.parts/%d.part", rootDataPath, f.creatorId, f.fileId, i) + b, err := ioutil.ReadFile(filePath) + if err != nil { + err = fmt.Errorf("path not exists: %s", filePath) + return "", err + } + md5Hash.Write(b) + } + return fmt.Sprintf("%x", md5Hash.Sum(nil)), nil +} + +func (f *cacheFile) ReadFileParts(fileParts int32, cb func(int, []byte)) (err error) { + if cb == nil { + return nil + } + + for i := 0; i < int(fileParts); i++ { + filePath := fmt.Sprintf("%s/0/%d.%d.parts/%d.part", rootDataPath, f.creatorId, f.fileId, i) + b, err := ioutil.ReadFile(filePath) + if err != nil { + err = fmt.Errorf("read %s error: %v", filePath, err) + return err + } + cb(i, b) + } + return nil +} + +func ReadFileOffsetData(filePath string, offset int32, limit int32) ([]byte, error) { + fileSize := getFileSize(filePath) + + if int64(offset) > fileSize { + limit = 0 + } else if int64(offset+limit) > fileSize { + limit = int32(fileSize - int64(offset)) + } + + f2, err := os.Open(filePath) + if err != nil { + glog.Error("open ", filePath, " error: ", err) + return nil, err + } + defer f2.Close() + + bytes := make([]byte, limit) + _, err = f2.ReadAt(bytes, int64(offset)) + if err != nil { + glog.Error("read file ", filePath, " error: ", err) + return nil, err + } + return bytes, nil +} diff --git a/service/nbfs/cachefs/document_file.go b/service/nbfs/cachefs/document_file.go new file mode 100644 index 00000000..ad5f4173 --- /dev/null +++ b/service/nbfs/cachefs/document_file.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cachefs + +import ( + "fmt" + "github.com/golang/glog" + "os" +) + +type DocumentFile struct { + fileId int64 + accessHash int64 + *os.File +} + +func CreateDocumentFile(fileId, accessHash int64) (d *DocumentFile, err error) { + d = &DocumentFile{fileId: fileId, accessHash: accessHash} + d.File, err = os.Create(d.ToFilePath()) + if err != nil { + glog.Error(err) + return nil, err + } + return d, nil +} + +func NewDocumentFile(fileId, accessHash int64) *DocumentFile { + return &DocumentFile{fileId: fileId, accessHash: accessHash} +} + +func (f *DocumentFile) ToFilePath() string { + return fmt.Sprintf("%s/0/%d.%d.dat", rootDataPath, f.fileId, f.accessHash) +} + +func (f *DocumentFile) ToFilePath2() string { + return fmt.Sprintf("/0/%d.%d.dat", f.fileId, f.accessHash) +} + +func (f *DocumentFile) Write(b []byte) (int, error) { + if f.File == nil { + return 0, fmt.Errorf("file not open") + } + + return f.File.Write(b) +} + +func (f *DocumentFile) Sync() { + if f.File == nil { + f.File.Sync() + } +} + +func (f *DocumentFile) Close() { + if f.File != nil { + f.File.Close() + } +} + +func (f *DocumentFile) ReadData(offset int32, limit int32) ([]byte, error) { + return ReadFileOffsetData(f.ToFilePath(), offset, limit) +} + diff --git a/service/nbfs/cachefs/photo_file.go b/service/nbfs/cachefs/photo_file.go new file mode 100644 index 00000000..5b2bbc63 --- /dev/null +++ b/service/nbfs/cachefs/photo_file.go @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cachefs + +import ( + "github.com/golang/glog" + "github.com/disintegration/imaging" + "fmt" + "bytes" + "image" + "os" + "io/ioutil" + "strings" +) + +const ( + kPhotoSizeOriginalType = "0" // client upload original photo + kPhotoSizeSmallType = "s" + kPhotoSizeMediumType = "m" + kPhotoSizeXLargeType = "x" + kPhotoSizeYLargeType = "y" + kPhotoSizeAType = "a" + kPhotoSizeBType = "b" + kPhotoSizeCType = "c" + + kPhotoSizeOriginalSize = 0 // client upload original photo + kPhotoSizeSmallSize = 90 + kPhotoSizeMediumSize = 320 + kPhotoSizeXLargeSize = 800 + kPhotoSizeYLargeSize = 1280 + kPhotoSizeASize = 160 + kPhotoSizeBSize = 320 + kPhotoSizeCSize = 640 + + kPhotoSizeAIndex = 4 +) + +var sizeList = []int{ + kPhotoSizeOriginalSize, + kPhotoSizeSmallSize, + kPhotoSizeMediumSize, + kPhotoSizeXLargeSize, + kPhotoSizeYLargeSize, + kPhotoSizeASize, + kPhotoSizeBSize, + kPhotoSizeCSize, +} + +func getSizeType(idx int) string { + switch idx { + case 0: + return kPhotoSizeOriginalType + case 1: + return kPhotoSizeSmallType + case 2: + return kPhotoSizeMediumType + case 3: + return kPhotoSizeXLargeType + case 4: + return kPhotoSizeYLargeType + case 5: + return kPhotoSizeAType + case 6: + return kPhotoSizeBType + case 7: + return kPhotoSizeCType + } + + return "" +} + +type resizeInfo struct { + isWidth bool + size int +} + +func makeResizeInfo(img image.Image) resizeInfo { + w := img.Bounds().Dx() + h := img.Bounds().Dy() + + if w >= h { + return resizeInfo{ + isWidth: true, + size: w, + } + } else { + return resizeInfo{ + isWidth: false, + size: h, + } + } +} + +type PhotoFile struct { + volumeId int64 + localId int32 + secretId int64 +} + +func NewPhotoFile(volumeId int64, localId int32, secretId int64) *PhotoFile { + return &PhotoFile{volumeId, localId, secretId} +} + +func (f *PhotoFile) ToFilePath() string { + return fmt.Sprintf("%s/%s/%d.%d.dat", rootDataPath, getSizeType(int(f.localId)), f.volumeId, f.secretId) +} + +func (f *PhotoFile) ToFilePath2() string { + return fmt.Sprintf("/%s/%d.%d.dat", getSizeType(int(f.localId)), f.volumeId, f.secretId) +} + +func (f *PhotoFile) WritePhotoFile(b []byte) error { + return ioutil.WriteFile(f.ToFilePath(), b, 0644) +} + +func (f *PhotoFile) ReadData(offset int32, limit int32) ([]byte, error) { + return ReadFileOffsetData(f.ToFilePath(), offset, limit) +} + +type PhotoInfo struct { + LocalId int32 + Width int32 + Height int32 + FileSize int64 +} + +// Save saves the image to file with the specified filename. +// The format is determined from the filename extension: "jpg" (or "jpeg"), "png", "gif", "tif" (or "tiff") and "bmp" are supported. +// +// Examples: +// +// // Save the image as PNG. +// err := imaging.Save(img, "out.png") +// +// // Save the image as JPEG with optional quality parameter set to 80. +// err := imaging.Save(img, "out.jpg", imaging.JPEGQuality(80)) +// +func saveImage(img image.Image, filename string, f imaging.Format, opts ...imaging.EncodeOption) (err error) { + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + return imaging.Encode(file, img, f, opts...) +} + +func getImageFormat(extName string) (int, error) { + formats := map[string]imaging.Format{ + ".jpg": imaging.JPEG, + ".jpeg": imaging.JPEG, + ".png": imaging.PNG, + ".tif": imaging.TIFF, + ".tiff": imaging.TIFF, + ".bmp": imaging.BMP, + ".gif": imaging.GIF, + } + + ext := strings.ToLower(extName) + f, ok := formats[ext] + if !ok { + return -1, imaging.ErrUnsupportedFormat + } + + return int(f), nil +} + +// TODO(@benqi): +// 我们未来的图片存储系统可能会按facebook的Haystack论文来实现 +// mtproto协议也定义了一套自己的文件存储方案，fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation; +// 在这里，我们重新定义mtproto的volume_id和local_id，对应Haystack的key和alternate_key，secret对应cookie +// 在当前简单实现里，volume_id由sonwflake生成，local_id对应于图片类型，secret为access_hash +// TODO(@benqi): +// 参数使用mtproto.File +func DoUploadedPhotoFile(src *PhotoFile, extName string, srcData []byte, isABC bool, cb func(pi *PhotoInfo)) error { + img, err := imaging.Decode(bytes.NewReader(srcData)) + if err != nil { + glog.Errorf("Decode %s error: {%v}", src, err) + return err + } + + pf := &PhotoFile{ + volumeId: src.volumeId, + localId: src.localId, + // secretId: src.secretId, + } + + imgSz := makeResizeInfo(img) + for i, sz := range sizeList { + pf.localId = int32(i) + if i != 0 { + if isABC { + if i <= kPhotoSizeAIndex { + continue + } + } else { + if i > kPhotoSizeAIndex { + continue + } + } + } + + pi := &PhotoInfo{ + LocalId: int32(pf.localId), + } + + if i == 0 { + err = pf.WritePhotoFile(srcData) + if err != nil { + glog.Errorf("encode error: {%v}", err) + return err + } + // pi.LocalId = int32(0) + pi.Width = int32(img.Bounds().Dx()) + pi.Height = int32(img.Bounds().Dy()) + pi.FileSize = int64(len(srcData)) + } else { + var dst *image.NRGBA + if imgSz.isWidth { + dst = imaging.Resize(img, sz, 0, imaging.Lanczos) + } else { + dst = imaging.Resize(img, 0, sz, imaging.Lanczos) + } + + f, err := getImageFormat(extName) + if err != nil { + glog.Error(err) + return err + } + + dstFileName := pf.ToFilePath() + if f == int(imaging.JPEG) { + err = saveImage(dst, dstFileName, imaging.JPEG, imaging.JPEGQuality(80)) + } else { + err = saveImage(dst, dstFileName, imaging.Format(f)) + } + // err = imaging.Save(dst, pf.ToFilePath()) + if err != nil { + glog.Errorf("encode error: {%v}", err) + return err + } + + pi.Width = int32(dst.Bounds().Dx()) + pi.Height = int32(dst.Bounds().Dy()) + pi.FileSize = getFileSize(dstFileName) + } + + if cb != nil { + cb(pi) + } + } + + return nil +} + diff --git a/server/nbfs/biz/core/file/test001.jpeg b/service/nbfs/cachefs/test001.jpeg similarity index 100% rename from server/nbfs/biz/core/file/test001.jpeg rename to service/nbfs/cachefs/test001.jpeg diff --git a/server/nbfs/biz/core/file/test002.jpeg b/service/nbfs/cachefs/test002.jpeg similarity index 100% rename from server/nbfs/biz/core/file/test002.jpeg rename to service/nbfs/cachefs/test002.jpeg diff --git a/service/nbfs/nbfs/local_nbfs_facade.go b/service/nbfs/nbfs/local_nbfs_facade.go new file mode 100644 index 00000000..d703524d --- /dev/null +++ b/service/nbfs/nbfs/local_nbfs_facade.go @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nbfs_client + +import ( + "github.com/golang/glog" + "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/service/nbfs/proto" + "fmt" + "hash" + "crypto/md5" + "github.com/nebulaim/telegramd/service/nbfs/cachefs" + "path" + "strings" + "github.com/nebulaim/telegramd/service/idgen/client" + "math/rand" + "os" + "time" +) + +type localNbfsFacade struct { + idgen.UUIDGen +} + +func localNbfsFacadeInstance() NbfsFacade { + return &localNbfsFacade{} +} + +func NewLocalNbfsFacade(dbName string) (*localNbfsFacade, error) { + var err error + + facade := &localNbfsFacade{} + // facade.ContactModel, err = core.InitContactModel(dbName) + + return facade, err +} + +func getFileExtName(filePath string) string { + var ext = path.Ext(filePath) + return strings.ToLower(ext) +} + +func getStorageFileTypeConstructor(extName string) int32 { + var c mtproto.TLConstructor + switch extName { + case ".partial": + c = mtproto.TLConstructor_CRC32_storage_filePartial + case ".jpeg", ".jpg": + c = mtproto.TLConstructor_CRC32_storage_fileJpeg + case ".gif": + c = mtproto.TLConstructor_CRC32_storage_fileGif + case ".png": + c = mtproto.TLConstructor_CRC32_storage_filePng + case ".pdf": + c = mtproto.TLConstructor_CRC32_storage_filePdf + case ".mp3": + c = mtproto.TLConstructor_CRC32_storage_fileMp3 + case ".mov": + c = mtproto.TLConstructor_CRC32_storage_fileMov + case ".mp4": + c = mtproto.TLConstructor_CRC32_storage_fileMp4 + case ".webp": + c = mtproto.TLConstructor_CRC32_storage_fileWebp + default: + // fileType.Constructor = mtproto.TLConstructor_CRC32_storage_fileUnknown + c = mtproto.TLConstructor_CRC32_storage_filePartial + } + return int32(c) +} + +func makeStorageFileType(c int32) *mtproto.Storage_FileType { + fileType := &mtproto.Storage_FileType{Data2: &mtproto.Storage_FileType_Data{}} + + switch mtproto.TLConstructor(c) { + case mtproto.TLConstructor_CRC32_storage_filePartial, + mtproto.TLConstructor_CRC32_storage_fileJpeg, + mtproto.TLConstructor_CRC32_storage_fileGif, + mtproto.TLConstructor_CRC32_storage_filePng, + mtproto.TLConstructor_CRC32_storage_filePdf, + mtproto.TLConstructor_CRC32_storage_fileMp3, + mtproto.TLConstructor_CRC32_storage_fileMov, + mtproto.TLConstructor_CRC32_storage_fileMp4, + mtproto.TLConstructor_CRC32_storage_fileWebp: + + fileType.Constructor = mtproto.TLConstructor(c) + default: + fileType.Constructor = mtproto.TLConstructor_CRC32_storage_filePartial + } + return fileType +} + +func (c *localNbfsFacade) Initialize(config string) error { + glog.Info("localNbfsFacade - Initialize config: ", config) + + var err error + c.UUIDGen, err = idgen.NewUUIDGen("snowflake", config) + if err != nil { + glog.Fatal("uuidgen init error: ", err) + } + + // dbName := config + // c.ContactModel, err = core.InitContactModel(dbName) + + return err +} + +func (c *localNbfsFacade) UploadPhotoFile(creatorId int64, file *mtproto.InputFile) (fileMDList []*nbfs.PhotoFileMetadata, err error) { + var ( + inputFile = file.GetData2() + md5Hash hash.Hash + fileSize = int64(0) + ) + + var cacheData []byte + + if inputFile.GetMd5Checksum() != "" { + md5Hash = md5.New() + } + cacheFile := cachefs.NewCacheFile(creatorId, inputFile.GetId()) + err = cacheFile.ReadFileParts(inputFile.GetParts(), func(i int, bytes []byte) { + if md5Hash != nil { + md5Hash.Write(bytes) + } + cacheData = append(cacheData, bytes...) + fileSize += int64(len(bytes)) + }) + if err != nil { + glog.Error(err) + return nil, err + } + + if fmt.Sprintf("%x", md5Hash.Sum(nil)) != inputFile.GetMd5Checksum() { + err = fmt.Errorf("invalid md5") + return nil, err + } + + photoId, _ := c.UUIDGen.GetUUID() + // accessHash := rand.Int63() + photoFile2 := cachefs.NewPhotoFile(photoId, 0, 0) + + ext := getFileExtName(inputFile.GetName()) + extType := getStorageFileTypeConstructor(ext) + + err = cachefs.DoUploadedPhotoFile(photoFile2, ext, cacheData, false, func(pi *cachefs.PhotoInfo) { + // 有点难理解，主要是为了不在这里引入snowflake + // ext := getFileExtName(inputFile.GetName()) + // extType := getStorageFileTypeConstructor(ext) + secretId := int64(extType) << 32 | int64(rand.Uint32()) + + srcFile := cachefs.NewPhotoFile(photoId, pi.LocalId, 0) + dstFile := cachefs.NewPhotoFile(photoId, pi.LocalId, secretId) + os.Rename(srcFile.ToFilePath(), dstFile.ToFilePath()) + + fileMD := &nbfs.PhotoFileMetadata{ + FileId: inputFile.GetId(), + PhotoId: photoId, + DcId: 2, + VolumeId: photoId, + LocalId: pi.LocalId, + SecretId: secretId, + Width: pi.Width, + Height: pi.Height, + FileSize: int32(pi.FileSize), + FilePath: dstFile.ToFilePath2(), + Ext: ext, + } + fileMDList = append(fileMDList, fileMD) + }) + if err != nil { + glog.Error(err) + return nil, err + } + + return +} + +func (c *localNbfsFacade) UploadProfilePhotoFile(creatorId int64, file *mtproto.InputFile) (fileMDList []*nbfs.PhotoFileMetadata, err error) { + var ( + inputFile = file.GetData2() + md5Hash hash.Hash + fileSize = int64(0) + ) + + var cacheData []byte + + if inputFile.GetMd5Checksum() != "" { + md5Hash = md5.New() + } + cacheFile := cachefs.NewCacheFile(creatorId, inputFile.GetId()) + err = cacheFile.ReadFileParts(inputFile.GetParts(), func(i int, bytes []byte) { + if md5Hash != nil { + md5Hash.Write(bytes) + } + cacheData = append(cacheData, bytes...) + fileSize += int64(len(bytes)) + }) + if err != nil { + glog.Error(err) + return nil, err + } + + if fmt.Sprintf("%x", md5Hash.Sum(nil)) != inputFile.GetMd5Checksum() { + err = fmt.Errorf("invalid md5") + return nil, err + } + + photoId, _ := c.UUIDGen.GetUUID() + // accessHash := rand.Int63() + photoFile2 := cachefs.NewPhotoFile(photoId, 0, 0) + + ext := getFileExtName(inputFile.GetName()) + extType := getStorageFileTypeConstructor(ext) + + err = cachefs.DoUploadedPhotoFile(photoFile2, ext, cacheData, false, func(pi *cachefs.PhotoInfo) { + // 有点难理解，主要是为了不在这里引入snowflake + secretId := int64(extType) << 32 | int64(rand.Uint32()) + + srcFile := cachefs.NewPhotoFile(photoId, pi.LocalId, 0) + dstFile := cachefs.NewPhotoFile(photoId, pi.LocalId, secretId) + os.Rename(srcFile.ToFilePath(), dstFile.ToFilePath()) + + fileMD := &nbfs.PhotoFileMetadata{ + FileId: inputFile.GetId(), + PhotoId: photoId, + DcId: 2, + VolumeId: photoId, + LocalId: pi.LocalId, + SecretId: secretId, + Width: pi.Width, + Height: pi.Height, + FileSize: int32(pi.FileSize), + FilePath: dstFile.ToFilePath2(), + Ext: ext, + } + fileMDList = append(fileMDList, fileMD) + }) + if err != nil { + glog.Error(err) + return nil, err + } + + return +} + +func (c *localNbfsFacade) UploadDocumentFile(creatorId int64, file *mtproto.InputFile) (fileMD *nbfs.DocumentFileMetadata, err error) { + var ( + inputFile = file.GetData2() + fileSize = int64(0) + documentFile *cachefs.DocumentFile + ) + + documentId, _ := c.UUIDGen.GetUUID() + + // 有点难理解，主要是为了不在这里引入snowflake + ext := getFileExtName(inputFile.GetName()) + extType := getStorageFileTypeConstructor(ext) + accessHash := int64(extType) << 32 | int64(rand.Uint32()) + + documentFile, err = cachefs.CreateDocumentFile(documentId, accessHash) + if err != nil { + return nil, err + } + defer documentFile.Close() + + // var cacheData []byte + cacheFile := cachefs.NewCacheFile(creatorId, inputFile.GetId()) + err = cacheFile.ReadFileParts(inputFile.GetParts(), func(i int, bytes []byte) { + // cacheData = append(cacheData, bytes...) + fileSize += int64(len(bytes)) + documentFile.Write(bytes) + documentFile.Sync() + }) + if err != nil { + glog.Error(err) + return + } + + fileMD = &nbfs.DocumentFileMetadata{ + FileId: inputFile.GetId(), + DocumentId: documentId, + AccessHash: accessHash, + DcId: 2, + FileSize: int32(fileSize), + FilePath: documentFile.ToFilePath2(), + UploadedFileName: inputFile.GetName(), + Ext: ext, + // MimeType: inputFile.GetName(), + } + return +} + +//func (c *localNbfsFacade) UploadFileParts(creatorId, filePartId int64) (bool, error) { +// return false, nil +//} +// + +func (c *localNbfsFacade) DownloadFile(location *mtproto.InputFileLocation, offset, limit int32) (file *mtproto.Upload_File, err error) { + var ( + // uploadFile *mtproto.Upload_File + // err error + bytes []byte + sType int32 + ) + + switch location.GetConstructor() { + case mtproto.TLConstructor_CRC32_inputFileLocation: + fileLocation := location.To_InputFileLocation() + file := cachefs.NewPhotoFile(fileLocation.GetVolumeId(), fileLocation.GetLocalId(), fileLocation.GetSecret()) + bytes, err = file.ReadData(offset, limit) + sType = int32(fileLocation.GetSecret() >> 32) + case mtproto.TLConstructor_CRC32_inputEncryptedFileLocation: + case mtproto.TLConstructor_CRC32_inputDocumentFileLocation: + fileLocation := location.To_InputDocumentFileLocation() + file := cachefs.NewDocumentFile(fileLocation.GetId(), fileLocation.GetAccessHash()) + bytes, err = file.ReadData(offset, limit) + sType = int32(fileLocation.GetAccessHash() >> 32) + case mtproto.TLConstructor_CRC32_inputDocumentFileLocationLayer11: + fileLocation := location.To_InputDocumentFileLocation() + file := cachefs.NewDocumentFile(fileLocation.GetId(), fileLocation.GetAccessHash()) + bytes, err = file.ReadData(offset, limit) + sType = int32(fileLocation.GetAccessHash() >> 32) + default: + err = fmt.Errorf("invalid InputFileLocation type: {%v}", location) + glog.Error(err) + return nil, err + } + + uploadFile := &mtproto.TLUploadFile{Data2: &mtproto.Upload_File_Data{ + Type: makeStorageFileType(sType), + Mtime: int32(time.Now().Unix()), + Bytes: bytes, + }} + + file = uploadFile.To_Upload_File() + return +} + +func init() { + Register("local", localNbfsFacadeInstance) +} diff --git a/service/nbfs/nbfs/nbfs.go b/service/nbfs/nbfs/nbfs.go new file mode 100644 index 00000000..a08b9fbd --- /dev/null +++ b/service/nbfs/nbfs/nbfs.go @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018, https://github.com/nebulaim + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nbfs_client + +import ( + "fmt" + "github.com/nebulaim/telegramd/proto/mtproto" + "github.com/nebulaim/telegramd/service/nbfs/proto" +) + +type NbfsFacade interface { + Initialize(config string) error + UploadPhotoFile(creatorId int64, file *mtproto.InputFile) ([]*nbfs.PhotoFileMetadata, error) + UploadProfilePhotoFile(creatorId int64, file *mtproto.InputFile) ([]*nbfs.PhotoFileMetadata, error) + UploadDocumentFile(creatorId int64, file *mtproto.InputFile) (*nbfs.DocumentFileMetadata, error) + // UploadFileParts(creatorId, filePartId int64) (bool, error) + DownloadFile(location *mtproto.InputFileLocation, offset, limit int32) (*mtproto.Upload_File, error) +} + +type Instance func() NbfsFacade + +var instances = make(map[string]Instance) + +func Register(name string, inst Instance) { + if inst == nil { + panic("register instance is nil") + } + if _, ok := instances[name]; ok { + panic("register called twice for instance " + name) + } + instances[name] = inst +} + +func NewNbfsFacade(name, config string) (inst NbfsFacade, err error) { + instanceFunc, ok := instances[name] + if !ok { + err = fmt.Errorf("unknown instance name %q (forgot to import?)", name) + return + } + inst = instanceFunc() + err = inst.Initialize(config) + if err != nil { + inst = nil + } + return +} diff --git a/server/nbfs/biz/dal/dataobject/file_parts_do.go b/service/nbfs/proto/nbfs.proto.go similarity index 53% rename from server/nbfs/biz/dal/dataobject/file_parts_do.go rename to service/nbfs/proto/nbfs.proto.go index 50e6e224..ccb4b033 100644 --- a/server/nbfs/biz/dal/dataobject/file_parts_do.go +++ b/service/nbfs/proto/nbfs.proto.go @@ -15,19 +15,34 @@ * limitations under the License. */ -package dataobject +package nbfs -type FilePartsDO struct { - Id int64 `db:"id"` - CreatorId int64 `db:"creator_id"` - CreatorUserId int32 `db:"creator_user_id"` - FileId int64 `db:"file_id"` - FilePartId int64 `db:"file_part_id"` - FilePart int32 `db:"file_part"` - IsBigFile int8 `db:"is_big_file"` - FileTotalParts int32 `db:"file_total_parts"` - FilePath string `db:"file_path"` - FileSize int64 `db:"file_size"` - CreatedAt string `db:"created_at"` - UpdatedAt string `db:"updated_at"` +type DocumentFileMetadata struct { + FileId int64 + DocumentId int64 + AccessHash int64 + DcId int32 + FileSize int32 + FilePath string + UploadedFileName string + Ext string + Md5Hash string + MimeType string +} + + +type PhotoFileMetadata struct { + FileId int64 + PhotoId int64 + PhotoType int8 + SizeType string + DcId int32 + VolumeId int64 + LocalId int32 + SecretId int64 + Width int32 + Height int32 + FileSize int32 + FilePath string + Ext string }