From 573c4737dabbe85c3f50dfa4a366bd34ee20d235 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 27 Jun 2024 20:22:24 +0530 Subject: [PATCH 01/98] remove vt mt wm and challenge workers --- .../blobbercore/allocation/connection.go | 4 -- .../allocation/file_changer_base.go | 5 -- .../blobbercore/challenge/worker.go | 4 +- .../blobbercore/filestore/storage.go | 57 +------------------ .../blobbercore/handler/handler_common.go | 15 ----- .../handler/object_operation_handler.go | 6 -- .../blobbercore/writemarker/worker.go | 22 +++---- 7 files changed, 14 insertions(+), 99 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index 06779a6e6..a0dd4f516 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -186,10 +186,6 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string if err != nil { return saveChange, err } - hasher := filestore.GetNewCommitHasher(contentSize) - change.hasher = hasher - change.seqPQ = seqpriorityqueue.NewSeqPriorityQueue(contentSize) - go hasher.Start(connectionObj.ctx, connectionID, connectionObj.AllocationID, fileName, pathHash, change.seqPQ) saveChange = true } else { change.lock.Lock() diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 337ee9be4..c1c9a67a5 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -8,7 +8,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) // BaseFileChanger base file change processor @@ -140,10 +139,6 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mute fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot fileInputData.ChunkSize = fc.ChunkSize fileInputData.Size = fc.Size - fileInputData.Hasher = GetHasher(fc.ConnectionID, encryption.Hash(fc.Path)) - if fileInputData.Hasher == nil { - return common.NewError("invalid_parameters", "Invalid parameters. Error getting hasher for commit.") - } _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) if err != nil { return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index e6fbe14e2..b4ef094d0 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -54,8 +54,8 @@ const batchSize = 5 // SetupWorkers start challenge workers func SetupWorkers(ctx context.Context) { - go startPullWorker(ctx) - go startWorkers(ctx) + // go startPullWorker(ctx) + // go startWorkers(ctx) } func startPullWorker(ctx context.Context) { diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 8febdaf40..5a6f0713e 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -264,40 +264,6 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) fileSize := rStat.Size() now := time.Now() - ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) - defer cancel() - err = fileData.Hasher.Wait(ctx, conID, allocID, fileData.Name, filePathHash) - if err != nil { - return false, common.NewError("hasher_wait_error", err.Error()) - } - elapsedWait := time.Since(now) - _, err = r.Seek(fileSize, io.SeekStart) - if err != nil { - return false, common.NewError("seek_error", err.Error()) - } - fmtRootBytes, err := fileData.Hasher.fmt.CalculateRootAndStoreNodes(r) - if err != nil { - return false, common.NewError("fmt_hash_calculation_error", err.Error()) - } - - validationRootBytes, err := fileData.Hasher.vt.CalculateRootAndStoreNodes(r, fileSize) - if err != nil { - return false, common.NewError("validation_hash_calculation_error", err.Error()) - } - fmtRoot := hex.EncodeToString(fmtRootBytes) - validationRoot := hex.EncodeToString(validationRootBytes) - elapsedRoot := time.Since(now) - elapsedWait - if fmtRoot != fileData.FixedMerkleRoot { - err = common.NewError("fixed_merkle_root_mismatch", - fmt.Sprintf("Expected %s got %s", fileData.FixedMerkleRoot, fmtRoot)) - return false, err - } - if validationRoot != fileData.ValidationRoot { - err = common.NewError("validation_root_mismatch", - "calculated validation root does not match with client's validation root") - return false, err - } - err = os.Rename(tempFilePath, preCommitPath) if err != nil { return false, common.NewError("write_error", err.Error()) @@ -314,7 +280,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) // 5. Move: It is Copy + Delete. Delete will not delete file if ref exists in database. i.e. copy would create // ref that refers to this file therefore it will be skipped fs.incrDecrAllocFileSizeAndNumber(allocID, fileSize, 1) - logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.Duration("elapsed_wait", elapsedWait), zap.Duration("elapsed_root", elapsedRoot), zap.Duration("elapsed_total", time.Since(now))) + logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.Duration("elapsed_total", time.Since(now))) return true, nil } @@ -508,7 +474,6 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes } startBlock := readBlockIn.StartBlockNum - endBlock := readBlockIn.StartBlockNum + readBlockIn.NumBlocks - 1 if startBlock < 0 { return nil, common.NewError("invalid_block_number", "Invalid block number. Start block number cannot be negative") @@ -551,26 +516,6 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes nodesSize := getNodesSize(filesize, util.MaxMerkleLeavesSize) vmp := &FileDownloadResponse{} - if readBlockIn.VerifyDownload { - vpOffset := int64(FMTSize) - if readBlockIn.FilestoreVersion == 1 { - vpOffset += readBlockIn.FileSize - } - vp := validationTreeProof{ - dataSize: readBlockIn.FileSize, - offset: vpOffset, - } - - logging.Logger.Debug("calling GetMerkleProofOfMultipleIndexes", zap.Any("readBlockIn", readBlockIn), zap.Any("vmp", vmp)) - nodes, indexes, err := vp.GetMerkleProofOfMultipleIndexes(file, nodesSize, startBlock, endBlock) - if err != nil { - return nil, common.NewError("get_merkle_proof_error", err.Error()) - } - - vmp.Nodes = nodes - vmp.Indexes = indexes - } - logging.Logger.Info("filestore_version", zap.Int("version", readBlockIn.FilestoreVersion)) fileOffset := int64(startBlock) * ChunkSize if readBlockIn.FilestoreVersion == 1 { _, err = file.Seek(fileOffset, io.SeekStart) diff --git a/code/go/0chain.net/blobbercore/handler/handler_common.go b/code/go/0chain.net/blobbercore/handler/handler_common.go index ebd6a9b64..8c9a46bbd 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_common.go +++ b/code/go/0chain.net/blobbercore/handler/handler_common.go @@ -8,8 +8,6 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/build" "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -127,10 +125,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu mutex := lock.GetMutex(allocation.Allocation{}.TableName(), allocationID) Logger.Info("Locking allocation", zap.String("allocation_id", allocationID)) - wmSet := writemarker.SetCommittingMarker(allocationID, true) - if !wmSet { - return nil, http.StatusBadRequest, common.NewError("pending_markers", "Committing marker set failed") - } mutex.Lock() defer mutex.Unlock() ctx = GetMetaDataStore().CreateTransaction(ctx) @@ -144,7 +138,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu if rollErr != nil { Logger.Error("couldn't rollback", zap.Error(err)) } - writemarker.SetCommittingMarker(allocationID, false) } }() @@ -160,14 +153,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu } Logger.Info("commit_success", zap.String("allocation_id", allocationID), zap.Any("response", resp)) - - if blobberRes, ok := resp.(*blobberhttp.CommitResult); ok { - // Save the write marker data - writemarker.SaveMarkerData(allocationID, blobberRes.WriteMarker.WM.Timestamp, blobberRes.WriteMarker.WM.ChainLength) - } else { - Logger.Error("Invalid response type for commit handler") - return resp, http.StatusInternalServerError, common.NewError("invalid_response_type", "Invalid response type for commit handler") - } return } } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 3fb471e9c..ee18c5306 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -540,12 +540,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b elapsedGetLock := time.Since(startTime) - elapsedAllocation - err = checkPendingMarkers(ctx, allocationObj.ID) - if err != nil { - Logger.Error("Error checking pending markers", zap.Error(err)) - return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed") - } - connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) if err != nil { // might be good to check if blobber already has stored writemarker diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index 7d5a3f924..7801a7c82 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -98,19 +98,19 @@ func deleteMarkerData(allocationID string) { // ) func SetupWorkers(ctx context.Context) { - var res []allocation.Res + // var res []allocation.Res - err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { - res = allocation.Repo.GetAllocationIds(ctx) - return nil - }) - if err != nil && err != gorm.ErrRecordNotFound { - logging.Logger.Error("error_getting_allocations_worker", - zap.Any("error", err)) - } + // err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + // res = allocation.Repo.GetAllocationIds(ctx) + // return nil + // }) + // if err != nil && err != gorm.ErrRecordNotFound { + // logging.Logger.Error("error_getting_allocations_worker", + // zap.Any("error", err)) + // } - startRedeem(ctx, res) - go startCollector(ctx) + // startRedeem(ctx, res) + // go startCollector(ctx) // go startCleanupWorker(ctx) } From 4178fc375f09ec9b13a0fc0ba106aa42a45bd283 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 1 Jul 2024 16:19:12 +0530 Subject: [PATCH 02/98] change apply changes --- .../allocation/allocationchange.go | 28 ++---- .../blobbercore/allocation/connection.go | 4 +- .../allocation/file_changer_base.go | 22 ++--- .../allocation/file_changer_update.go | 10 +- .../allocation/file_changer_upload.go | 93 +++---------------- .../allocation/file_changer_upload_main.go | 7 +- .../blobbercore/filestore/storage.go | 34 ++++++- .../0chain.net/blobbercore/filestore/store.go | 11 +-- .../blobbercore/filestore/tree_validation.go | 13 ++- .../handler/file_command_update.go | 15 ++- .../handler/file_command_upload.go | 25 +---- .../blobbercore/reference/dbCollector.go | 6 ++ .../0chain.net/blobbercore/reference/ref.go | 2 + 13 files changed, 102 insertions(+), 168 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 92130f257..ff2e65bdc 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -32,8 +32,8 @@ const ( type AllocationChangeProcessor interface { CommitToFileStore(ctx context.Context, mut *sync.Mutex) error DeleteTempFile() error - ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string, - ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) + ApplyChange(ctx context.Context, change *AllocationChange, + ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error GetPath() []string Marshal() (string, error) Unmarshal(string) error @@ -74,6 +74,7 @@ type AllocationChange struct { Input string `gorm:"column:input"` FilePath string `gorm:"-"` LookupHash string `gorm:"column:lookup_hash;size:64"` + AllocationID string `gorm:"-" json:"-"` datastore.ModelWithTS } @@ -255,28 +256,17 @@ func (cc *AllocationChangeCollector) ComputeProperties() { } func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocationRoot, prevAllocationRoot string, - ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - rootRef, err := cc.GetRootRef(ctx) - if err != nil { - return rootRef, err - } - if rootRef.Hash != prevAllocationRoot { - return rootRef, common.NewError("invalid_prev_root", "Invalid prev root") - } + ts common.Timestamp, fileIDMeta map[string]string) error { + collector := reference.NewCollector(len(cc.Changes)) for idx, change := range cc.Changes { + change.AllocationID = cc.AllocationID changeProcessor := cc.AllocationChanges[idx] - _, err := changeProcessor.ApplyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) + err := changeProcessor.ApplyChange(ctx, change, ts, fileIDMeta, collector) if err != nil { - return rootRef, err + return err } } - collector := reference.NewCollector(len(cc.Changes)) - _, err = rootRef.CalculateHash(ctx, true, collector) - if err != nil { - return rootRef, err - } - err = collector.Finalize(ctx) - return rootRef, err + return collector.Finalize(ctx) } func (a *AllocationChangeCollector) CommitToFileStore(ctx context.Context) error { diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index 8a11c58b0..dfc4077b2 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -68,10 +68,10 @@ func SaveFileChanger(connectionID string, fileChanger *BaseFileChanger) error { return common.NewError("connection_not_found", "connection not found") } connectionObj.lock.Lock() - if connectionObj.changes[fileChanger.PathHash] == nil { + if connectionObj.changes[fileChanger.LookupHash] == nil { return common.NewError("connection_change_not_found", "connection change not found") } - connectionObj.changes[fileChanger.PathHash].baseChanger = fileChanger + connectionObj.changes[fileChanger.LookupHash].baseChanger = fileChanger connectionObj.lock.Unlock() return nil } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index c1c9a67a5..81dd7b3fa 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -32,16 +32,12 @@ type BaseFileChanger struct { //client side: MimeType string `json:"mimetype,omitempty"` //client side: - //client side: - FixedMerkleRoot string `json:"fixed_merkle_root,omitempty"` //server side: update them by ChangeProcessor - AllocationID string `json:"allocation_id"` - //client side: - ValidationRootSignature string `json:"validation_root_signature,omitempty"` - //client side: - ValidationRoot string `json:"validation_root,omitempty"` - Size int64 `json:"size"` + AllocationID string `json:"allocation_id"` + DataHash string `json:"data_hash"` + DataHashSignature string `json:"data_hash_signature"` + Size int64 `json:"size"` //server side: ThumbnailHash string `json:"thumbnail_content_hash,omitempty"` ThumbnailSize int64 `json:"thumbnail_size"` @@ -58,7 +54,7 @@ type BaseFileChanger struct { ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of CompactMerkleTree ChunkHash string `json:"chunk_hash,omitempty"` UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to - PathHash string `json:"-"` // hash of path + LookupHash string `json:"-"` // hash of path } // swagger:model UploadResult @@ -106,7 +102,8 @@ func (fc *BaseFileChanger) DeleteTempFile() error { fileInputData := &filestore.FileInputData{} fileInputData.Name = fc.Filename fileInputData.Path = fc.Path - fileInputData.ValidationRoot = fc.ValidationRoot + fileInputData.DataHash = fc.DataHash + fileInputData.LookupHash = fc.LookupHash err := filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fc.ConnectionID, fileInputData) if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} @@ -125,6 +122,7 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mute fileInputData.Name = fc.ThumbnailFilename fileInputData.Path = fc.Path fileInputData.ThumbnailHash = fc.ThumbnailHash + fileInputData.LookupHash = fc.LookupHash fileInputData.ChunkSize = fc.ChunkSize fileInputData.IsThumbnail = true _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) @@ -135,8 +133,8 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mute fileInputData := &filestore.FileInputData{} fileInputData.Name = fc.Filename fileInputData.Path = fc.Path - fileInputData.ValidationRoot = fc.ValidationRoot - fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot + fileInputData.DataHash = fc.DataHash + fileInputData.LookupHash = fc.LookupHash fileInputData.ChunkSize = fc.ChunkSize fileInputData.Size = fc.Size _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index e1e1282db..3b94d3352 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -74,18 +74,14 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.HashToBeComputed = true nf.deleteHash = make(map[string]int) - if fileRef.ValidationRoot != "" && fileRef.ValidationRoot != nf.ValidationRoot { - nf.deleteHash[fileRef.ValidationRoot] = fileRef.FilestoreVersion - } - fileRef.ActualFileHash = nf.ActualHash fileRef.ActualFileHashSignature = nf.ActualFileHashSignature fileRef.ActualFileSize = nf.ActualSize fileRef.MimeType = nf.MimeType - fileRef.ValidationRootSignature = nf.ValidationRootSignature - fileRef.ValidationRoot = nf.ValidationRoot fileRef.CustomMeta = nf.CustomMeta - fileRef.FixedMerkleRoot = nf.FixedMerkleRoot + fileRef.DataHash = nf.DataHash + fileRef.DataHashSignature = nf.DataHashSignature + fileRef.LookupHash = nf.LookupHash fileRef.AllocationRoot = allocationRoot fileRef.Size = nf.Size fileRef.ThumbnailHash = nf.ThumbnailHash diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 8f2fd30f7..21c71b165 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -5,9 +5,7 @@ import ( "encoding/json" "fmt" "path/filepath" - "strings" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -22,93 +20,23 @@ type UploadFileChanger struct { } // ApplyChange update references, and create a new FileRef -func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - - totalRefs, err := reference.CountRefs(ctx, nf.AllocationID) - if err != nil { - return nil, err - } - - if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs { - return nil, common.NewErrorf("max_alloc_dir_files_reached", - "maximum files and directories already reached: %v", err) - } - - fields, err := common.GetPathFields(filepath.Dir(nf.Path)) - if err != nil { - return nil, err - } - if rootRef.CreatedAt == 0 { - rootRef.CreatedAt = ts - } - - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - - dirRef := rootRef - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - if child.Type != reference.DIRECTORY { - return nil, common.NewError("invalid_reference_path", "Reference path has invalid ref type") - } - dirRef = child - dirRef.UpdatedAt = ts - dirRef.HashToBeComputed = true - found = true - } - } - - if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir { - return nil, common.NewErrorf("max_objects_in_dir_reached", - "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir) - } - - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = dirRef.AllocationID - newRef.Path = "/" + strings.Join(fields[:i+1], "/") - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.FileID = fileID - newRef.ParentPath = "/" + strings.Join(fields[:i], "/") - newRef.Name = fields[i] - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - newRef.HashToBeComputed = true - - dirRef.AddChild(newRef) - dirRef = newRef - } - } - - for _, child := range dirRef.Children { - if child.Name == nf.Filename { - return nil, common.NewError("duplicate_file", "File already exists") - } - } +func (nf *UploadFileChanger) applyChange(ctx context.Context, + ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { + parentPath, _ := filepath.Split(nf.Path) + nf.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) newFile := &reference.Ref{ ActualFileHash: nf.ActualHash, ActualFileHashSignature: nf.ActualFileHashSignature, ActualFileSize: nf.ActualSize, - AllocationID: dirRef.AllocationID, - ValidationRoot: nf.ValidationRoot, - ValidationRootSignature: nf.ValidationRootSignature, + AllocationID: nf.AllocationID, CustomMeta: nf.CustomMeta, - FixedMerkleRoot: nf.FixedMerkleRoot, Name: nf.Filename, Path: nf.Path, - ParentPath: dirRef.Path, + ParentPath: parentPath, Type: reference.FILE, Size: nf.Size, MimeType: nf.MimeType, - AllocationRoot: allocationRoot, ThumbnailHash: nf.ThumbnailHash, ThumbnailSize: nf.ThumbnailSize, ActualThumbnailHash: nf.ActualThumbnailHash, @@ -120,19 +48,20 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference UpdatedAt: ts, HashToBeComputed: true, IsPrecommit: true, + LookupHash: nf.LookupHash, + DataHash: nf.DataHash, + DataHashSignature: nf.DataHashSignature, FilestoreVersion: filestore.VERSION, } fileID, ok := fileIDMeta[newFile.Path] if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", + return common.NewError("invalid_parameter", fmt.Sprintf("file path %s has no entry in fileID meta", newFile.Path)) } newFile.FileID = fileID - dirRef.AddChild(newFile) - - return rootRef, nil + return nil } // Marshal marshal and change to persistent to postgres diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go index 5a41a40aa..de700da46 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go @@ -5,11 +5,12 @@ package allocation import ( "context" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) +func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *AllocationChange, + ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) (*reference.Ref, error) { + return nf.applyChange(ctx, change, ts, fileIDMeta, collector) } diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index a4d832c5f..0d2d1d77c 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -57,6 +57,7 @@ const ( MerkleChunkSize = 64 ChunkSize = 64 * KB BufferSize = 80 * ChunkSize + ThumbnailSuffix = "_thumbnail" ) func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, infile multipart.File) (*FileOutputData, error) { @@ -157,6 +158,17 @@ func (fs *FileStore) MoveToFilestore(allocID, hash string, version int) error { } _ = os.Rename(preCommitPath, fPath) + + // Check if thumbnail exists + thumbPath := fs.getPreCommitPathForFile(allocID, hash+ThumbnailSuffix, version) + if _, err := os.Stat(thumbPath); err == nil { + thumbFilePath, err := fs.GetPathForFile(allocID, hash+ThumbnailSuffix, version) + if err != nil { + return common.NewError("get_file_path_error", err.Error()) + } + _ = os.Rename(thumbPath, thumbFilePath) + } + return nil } @@ -193,14 +205,14 @@ func (fs *FileStore) DeletePreCommitDir(allocID string) error { } func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) (_ bool, err error) { - + now := time.Now() logging.Logger.Info("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData)) filePathHash := encryption.Hash(fileData.Path) tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, filePathHash, conID) - fileHash := fileData.ValidationRoot + fileHash := fileData.LookupHash if fileData.IsThumbnail { - fileHash = fileData.ThumbnailHash + fileHash = fileData.LookupHash + ThumbnailSuffix } preCommitPath := fs.getPreCommitPathForFile(allocID, fileHash, VERSION) @@ -255,7 +267,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) return true, nil } - key := getKey(allocID, fileData.ValidationRoot) + key := getKey(allocID, fileData.LookupHash) l, _ := contentHashMapLock.GetLock(key) l.Lock() defer func() { @@ -270,7 +282,18 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) } fileSize := rStat.Size() - now := time.Now() + ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) + defer cancel() + err = fileData.Hasher.Wait(ctx, conID, allocID, fileData.Name, filePathHash) + if err != nil { + return false, common.NewError("hasher_wait_error", err.Error()) + } + md5Hash := fileData.Hasher.GetMd5Hash() + if md5Hash != fileData.DataHash { + return false, common.NewError("hash_mismatch", + fmt.Sprintf("calculated hash does not match with expected hash. Expected %s, got %s.", + fileData.DataHash, md5Hash)) + } err = os.Rename(tempFilePath, preCommitPath) if err != nil { return false, common.NewError("write_error", err.Error()) @@ -401,6 +424,7 @@ func (fs *FileStore) DeleteAllocation(allocID string) { func (fs *FileStore) GetFileThumbnail(readBlockIn *ReadBlockInput) (*FileDownloadResponse, error) { var fileObjectPath string var err error + readBlockIn.Hash += ThumbnailSuffix startBlock := readBlockIn.StartBlockNum if startBlock < 0 { return nil, common.NewError("invalid_block_number", "Invalid block number. Start block number cannot be negative") diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index b123e6292..069910cb6 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -10,12 +10,11 @@ const ( ) type FileInputData struct { - Name string - Path string - ValidationRoot string - FixedMerkleRoot string - ThumbnailHash string - + Name string + Path string + DataHash string + ThumbnailHash string + LookupHash string // ChunkSize chunk size ChunkSize int64 //UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 75856d485..60012ec70 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -5,9 +5,11 @@ package filestore import ( "context" + "crypto/md5" "encoding/hex" "errors" "fmt" + "hash" "io" "math" "os" @@ -406,6 +408,7 @@ func getNewValidationTree(dataSize int64) *validationTree { type CommitHasher struct { fmt *fixedMerkleTree vt *validationTree + md5hasher hash.Hash isInitialized bool doneChan chan struct{} hashErr error @@ -414,8 +417,7 @@ type CommitHasher struct { func GetNewCommitHasher(dataSize int64) *CommitHasher { c := new(CommitHasher) - c.fmt = getNewFixedMerkleTree() - c.vt = getNewValidationTree(dataSize) + c.md5hasher = md5.New() c.isInitialized = true c.doneChan = make(chan struct{}) c.dataSize = dataSize @@ -474,7 +476,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil pq.DataBytes -= int64(n) pq.Offset += int64(n) totalWritten += int64(n) - _, err = c.Write(buf[:n]) + _, err = c.md5hasher.Write(buf[:n]) if err != nil { logging.Logger.Error("hasher_write", zap.Error(err), zap.Int("n", n), zap.Int64("offset", pq.Offset), zap.Int64("dataBytes", pq.DataBytes), zap.Int64("dataSize", c.dataSize), zap.String("filename", fileName), zap.Int64("totalWritten", totalWritten)) c.hashErr = err @@ -483,7 +485,6 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil } buf = nil if toFinalize { - c.hashErr = c.Finalize() return } } @@ -568,3 +569,7 @@ func (c *CommitHasher) GetFixedMerkleRoot() string { func (c *CommitHasher) GetValidationMerkleRoot() string { return hex.EncodeToString(c.vt.GetValidationRoot()) } + +func (c *CommitHasher) GetMd5Hash() string { + return hex.EncodeToString(c.md5hasher.Sum(nil)) +} diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index 1b1daf5d4..d5579c40d 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -14,7 +14,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" sdkConst "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/fileref" ) @@ -72,19 +71,19 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_connection", "Invalid connection id") } - cmd.fileChanger.PathHash = encryption.Hash(cmd.fileChanger.Path) + cmd.fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, cmd.fileChanger.Path) if cmd.fileChanger.ChunkSize <= 0 { cmd.fileChanger.ChunkSize = fileref.CHUNK_SIZE } - cmd.existingFileRef = allocation.GetExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + cmd.existingFileRef = allocation.GetExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash) if cmd.existingFileRef == nil { cmd.existingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.fileChanger.Path) if cmd.existingFileRef == nil { return common.NewError("invalid_file_update", "File at path does not exist for update") } - allocation.SaveExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash, cmd.existingFileRef) //nolint:errcheck + allocation.SaveExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash, cmd.existingFileRef) //nolint:errcheck } thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) @@ -112,7 +111,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj result.Filename = cmd.fileChanger.Filename defer cmd.contentFile.Close() - filePathHash := cmd.fileChanger.PathHash + filePathHash := cmd.fileChanger.LookupHash connID := cmd.fileChanger.ConnectionID fileInputData := &filestore.FileInputData{ @@ -156,7 +155,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj } } - saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size) + saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.LookupHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size) if err != nil { return result, err } @@ -182,7 +181,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat connectionID := cmd.fileChanger.ConnectionID if cmd.thumbHeader != nil { defer cmd.thumbFile.Close() - thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash} + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.LookupHash} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) @@ -196,7 +195,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat } func (cmd *UpdateFileCommand) reloadChange() { - changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash) if changer != nil && changer.ThumbnailHash != "" { cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 88fea77f9..0ae212471 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -17,7 +17,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/fileref" @@ -81,21 +80,7 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_connection", "Invalid connection id") } - fileChanger.PathHash = encryption.Hash(fileChanger.Path) - - if fileChanger.UploadOffset == 0 { - isExist, err := reference.IsRefExist(ctx, allocationObj.ID, fileChanger.Path) - - if err != nil { - logging.Logger.Error(err.Error()) - return common.NewError("database_error", "Got db error while getting ref") - } - - if isExist { - msg := fmt.Sprintf("File at path :%s: already exists", fileChanger.Path) - return common.NewError("duplicate_file", msg) - } - } + fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, cmd.fileChanger.Path) thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) if thumbHeader != nil { @@ -135,7 +120,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj ChunkSize: cmd.fileChanger.ChunkSize, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, - FilePathHash: cmd.fileChanger.PathHash, + FilePathHash: cmd.fileChanger.LookupHash, Size: cmd.fileChanger.Size, } fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, fileInputData, cmd.contentFile) @@ -171,7 +156,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj } } - saveChange, err := allocation.SaveFileChange(ctx, connectionID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size) + saveChange, err := allocation.SaveFileChange(ctx, connectionID, cmd.fileChanger.LookupHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size) if err != nil { logging.Logger.Error("UploadFileCommand.ProcessContent", zap.Error(err)) return result, err @@ -200,7 +185,7 @@ func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat if cmd.thumbHeader != nil { defer cmd.thumbFile.Close() - thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash} + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.LookupHash} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) @@ -214,7 +199,7 @@ func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat } func (cmd *UploadFileCommand) reloadChange() { - changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash) if changer != nil && changer.ThumbnailHash != "" { cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index 37c3cc808..9c3c9b61e 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -2,6 +2,7 @@ package reference import ( "context" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" ) @@ -15,6 +16,7 @@ type QueryCollector interface { type dbCollector struct { createdRefs []*Ref deletedRefs []*Ref + mut sync.Mutex } func NewCollector(changes int) QueryCollector { @@ -25,11 +27,15 @@ func NewCollector(changes int) QueryCollector { } func (dc *dbCollector) CreateRefRecord(ref *Ref) { + dc.mut.Lock() dc.createdRefs = append(dc.createdRefs, ref) + dc.mut.Unlock() } func (dc *dbCollector) DeleteRefRecord(ref *Ref) { + dc.mut.Lock() dc.deletedRefs = append(dc.deletedRefs, ref) + dc.mut.Unlock() } func (dc *dbCollector) Finalize(ctx context.Context) error { diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 3f844baed..e5d6ae223 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -89,6 +89,8 @@ type Ref struct { NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"` NumBlockDownloads int64 `gorm:"column:num_of_block_downloads" json:"num_of_block_downloads"` FilestoreVersion int `gorm:"column:filestore_version" json:"-"` + DataHash string `gorm:"column:data_hash" filelist:"data_hash"` + DataHashSignature string `gorm:"column:data_hash_signature" filelist:"data_hash_signature"` HashToBeComputed bool `gorm:"-"` prevID int64 `gorm:"-"` } From 995912dd42f501a7599d98999feb1b97fdc37090 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Mon, 1 Jul 2024 17:54:52 +0530 Subject: [PATCH 03/98] support upload,delete and createdir for enterprise --- .../allocation/allocationchange.go | 105 +-- .../blobbercore/allocation/connection.go | 6 + .../allocation/deletefilechange.go | 12 +- .../allocation/file_changer_upload.go | 29 +- .../allocation/file_changer_upload_main.go | 6 +- .../blobbercore/allocation/newdirchange.go | 92 +-- .../blobbercore/filestore/storage.go | 8 + .../handler/object_operation_handler.go | 730 ++++++++---------- .../0chain.net/blobbercore/handler/worker.go | 7 +- .../blobbercore/reference/object.go | 63 +- .../0chain.net/blobbercore/reference/ref.go | 16 + 11 files changed, 439 insertions(+), 635 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index ff2e65bdc..94d871a37 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -32,7 +32,7 @@ const ( type AllocationChangeProcessor interface { CommitToFileStore(ctx context.Context, mut *sync.Mutex) error DeleteTempFile() error - ApplyChange(ctx context.Context, change *AllocationChange, + ApplyChange(ctx context.Context, ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error GetPath() []string Marshal() (string, error) @@ -231,18 +231,18 @@ func (cc *AllocationChangeCollector) ComputeProperties() { switch change.Operation { case constants.FileOperationInsert: acp = new(UploadFileChanger) - case constants.FileOperationUpdate: - acp = new(UpdateFileChanger) + // case constants.FileOperationUpdate: + // acp = new(UpdateFileChanger) case constants.FileOperationDelete: acp = new(DeleteFileChange) - case constants.FileOperationRename: - acp = new(RenameFileChange) - case constants.FileOperationCopy: - acp = new(CopyFileChange) + // case constants.FileOperationRename: + // acp = new(RenameFileChange) + // case constants.FileOperationCopy: + // acp = new(CopyFileChange) case constants.FileOperationCreateDir: acp = new(NewDir) - case constants.FileOperationMove: - acp = new(MoveFileChange) + // case constants.FileOperationMove: + // acp = new(MoveFileChange) } if acp == nil { @@ -255,13 +255,13 @@ func (cc *AllocationChangeCollector) ComputeProperties() { } } -func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocationRoot, prevAllocationRoot string, +func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, ts common.Timestamp, fileIDMeta map[string]string) error { collector := reference.NewCollector(len(cc.Changes)) for idx, change := range cc.Changes { change.AllocationID = cc.AllocationID changeProcessor := cc.AllocationChanges[idx] - err := changeProcessor.ApplyChange(ctx, change, ts, fileIDMeta, collector) + err := changeProcessor.ApplyChange(ctx, ts, fileIDMeta, collector) if err != nil { return err } @@ -294,12 +294,9 @@ func (a *AllocationChangeCollector) DeleteChanges(ctx context.Context) { } type Result struct { - Id string - ValidationRoot string - PrevValidationRoot string - ThumbnailHash string - PrevThumbnailHash string - FilestoreVersion int + Id string + LookupHash string + FilestoreVersion int } // TODO: Need to speed up this function @@ -316,18 +313,11 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { tx := datastore.GetStore().GetTransaction(ctx) - err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash", "filestore_version").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). + err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "filestore_version").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error { for _, ref := range refs { - var count int64 - if ref.PrevValidationRoot != "" { - tx.Model(&reference.Ref{}). - Where("allocation_id=? AND validation_root=?", a.AllocationID, ref.PrevValidationRoot). - Count(&count) - } - limitCh <- struct{}{} wg.Add(1) @@ -337,32 +327,9 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { wg.Done() }() - if count == 0 && ref.PrevValidationRoot != "" { - err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevValidationRoot, ref.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), - zap.String("validation_root", ref.ValidationRoot)) - } - } - err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ValidationRoot, ref.FilestoreVersion) + err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.LookupHash, ref.FilestoreVersion) if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error()), - zap.String("validation_root", ref.ValidationRoot)) - } - - if ref.ThumbnailHash != "" && ref.ThumbnailHash != ref.PrevThumbnailHash { - if ref.PrevThumbnailHash != "" { - err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevThumbnailHash, ref.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail file: %s", err.Error()), - zap.String("thumbnail_hash", ref.ThumbnailHash)) - } - } - err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ThumbnailHash, ref.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while moving thumbnail file: %s", err.Error()), - zap.String("thumbnail_hash", ref.ThumbnailHash)) - } + logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error())) } }(ref) @@ -378,7 +345,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { return err } - return tx.Exec("UPDATE reference_objects SET is_precommit=?, prev_validation_root=validation_root, prev_thumbnail_hash=thumbnail_hash WHERE allocation_id=? AND is_precommit=? AND deleted_at is NULL", false, a.AllocationID, true).Error + return tx.Exec("UPDATE reference_objects SET is_precommit=? WHERE allocation_id=? AND is_precommit=? AND deleted_at is NULL", false, a.AllocationID, true).Error }) return err } @@ -391,47 +358,27 @@ func deleteFromFileStore(ctx context.Context, allocationID string) error { return datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) - err := db.Model(&reference.Ref{}).Unscoped().Select("id", "validation_root", "thumbnail_hash"). + err := db.Model(&reference.Ref{}).Unscoped().Select("id", "lookup_hash", "filestore_version"). Where("allocation_id=? AND is_precommit=? AND type=? AND deleted_at is not NULL", allocationID, true, reference.FILE). FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error { for _, res := range results { - var count int64 - tx.Model(&reference.Ref{}). - Where("allocation_id=? AND validation_root=?", allocationID, res.ValidationRoot). - Count(&count) - - if count != 0 && res.ThumbnailHash == "" { - continue - } - limitCh <- struct{}{} wg.Add(1) - go func(res Result, count int64) { + go func(res Result) { defer func() { <-limitCh wg.Done() }() - if count == 0 { - err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ValidationRoot, - res.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), - zap.String("validation_root", res.ValidationRoot)) - } - } - - if res.ThumbnailHash != "" { - err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ThumbnailHash, res.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail: %s", err.Error()), - zap.String("thumbnail", res.ThumbnailHash)) - } + err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.LookupHash, + res.FilestoreVersion) + if err != nil { + logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), + zap.String("validation_root", res.LookupHash)) } - - }(res, count) + }(res) } return nil diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index dfc4077b2..5b731e52b 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -207,6 +207,12 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string DataBytes: dataWritten, }, contentSize) if addSize != 0 { + //check if reference exists and get the size + existingSize, err := reference.GetObjectSizeByLookupHash(ctx, pathHash) + if err != nil { + return saveChange, err + } + addSize -= existingSize UpdateConnectionObjSize(connectionID, addSize) } } else { diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index 4caf0d1e3..c683f7edd 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -23,15 +23,9 @@ type DeleteFileChange struct { Hash string `json:"hash"` } -func (nf *DeleteFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { - - err := reference.DeleteObject(ctx, rootRef, nf.AllocationID, filepath.Clean(nf.Path), ts) - if err != nil { - return nil, err - } - - return nil, nil +func (nf *DeleteFileChange) ApplyChange(ctx context.Context, + ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + return reference.DeleteObject(ctx, nf.AllocationID, filepath.Clean(nf.Path), ts) } func (nf *DeleteFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 21c71b165..290d4f4a1 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -5,8 +5,11 @@ import ( "encoding/json" "fmt" "path/filepath" + "strings" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" @@ -23,6 +26,10 @@ type UploadFileChanger struct { func (nf *UploadFileChanger) applyChange(ctx context.Context, ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { + if nf.AllocationID == "" { + return common.NewError("invalid_parameter", "allocation_id is required") + } + parentPath, _ := filepath.Split(nf.Path) nf.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) newFile := &reference.Ref{ @@ -51,6 +58,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, LookupHash: nf.LookupHash, DataHash: nf.DataHash, DataHashSignature: nf.DataHashSignature, + PathLevel: len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")), FilestoreVersion: filestore.VERSION, } @@ -61,7 +69,26 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, } newFile.FileID = fileID - return nil + // find if ref exists + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + var refResult struct { + ID int64 + } + db := datastore.GetStore().GetTransaction(ctx) + err := db.Model(&reference.Ref{}).Select("id").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error + if err == gorm.ErrRecordNotFound { + collector.CreateRefRecord(newFile) + } else if err != nil { + return err + } + deleteRecord := &reference.Ref{ + ID: refResult.ID, + } + collector.DeleteRefRecord(deleteRecord) + return nil + }) + + return err } // Marshal marshal and change to persistent to postgres diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go index de700da46..324862949 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go @@ -10,7 +10,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *AllocationChange, - ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) (*reference.Ref, error) { - return nf.applyChange(ctx, change, ts, fileIDMeta, collector) +func (nf *UploadFileChanger) ApplyChange(ctx context.Context, + ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { + return nf.applyChange(ctx, ts, fileIDMeta, collector) } diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 7d630b0f6..d38e8135b 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -8,8 +8,7 @@ import ( "strings" "sync" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -21,69 +20,38 @@ type NewDir struct { AllocationID string `json:"allocation_id"` } -func (nf *NewDir) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - - totalRefs, err := reference.CountRefs(ctx, nf.AllocationID) - if err != nil { - return nil, err - } - - if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs { - return nil, common.NewErrorf("max_alloc_dir_files_reached", - "maximum files and directories already reached: %v", err) - } - - err = nf.Unmarshal(change.Input) - if err != nil { - return nil, err +func (nf *NewDir) ApplyChange(ctx context.Context, + ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { + + newRef := reference.NewDirectoryRef() + newRef.AllocationID = nf.AllocationID + newRef.Path = nf.Path + newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, newRef.Path) + newRef.PathLevel = len(strings.Split(strings.TrimRight(newRef.Path, "/"), "/")) + newRef.ParentPath = filepath.Dir(newRef.Path) + newRef.Name = filepath.Base(newRef.Path) + newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, newRef.Path) + newRef.CreatedAt = ts + newRef.UpdatedAt = ts + newRef.HashToBeComputed = true + fileID, ok := fileIDMeta[newRef.Path] + if !ok || fileID == "" { + return common.NewError("invalid_parameter", + fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) } - - if rootRef.CreatedAt == 0 { - rootRef.CreatedAt = ts - } - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - fields, err := common.GetPathFields(nf.Path) - if err != nil { - return nil, err - } - - dirRef := rootRef - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - dirRef = child - dirRef.HashToBeComputed = true - dirRef.UpdatedAt = ts - found = true - break - } + newRef.FileID = fileID + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + //check if ref exists + exists, err := reference.IsRefExist(ctx, nf.AllocationID, newRef.Path) + if err != nil { + return err } - - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = nf.AllocationID - newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/")) - newRef.PathLevel = len(fields) + 1 - newRef.ParentPath = filepath.Dir(newRef.Path) - newRef.Name = fields[i] - newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, newRef.Path) - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - newRef.HashToBeComputed = true - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.FileID = fileID - dirRef.AddChild(newRef) - dirRef = newRef + if !exists { + collector.CreateRefRecord(newRef) } - } - return rootRef, nil + return nil + }) + return err } func (nd *NewDir) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 0d2d1d77c..fc7cef810 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -191,6 +191,14 @@ func (fs *FileStore) DeleteFromFilestore(allocID, hash string, version int) erro } fs.incrDecrAllocFileSizeAndNumber(allocID, -stat.Size(), -1) + thumbPath, err := fs.GetPathForFile(allocID, hash+ThumbnailSuffix, version) + if err != nil { + return common.NewError("get_file_path_error", err.Error()) + } + if _, err := os.Stat(thumbPath); err == nil { + os.Remove(thumbPath) //nolint:errcheck + } + return nil } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index e23c545cf..a57fbc0fa 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -499,7 +499,6 @@ func (fsh *StorageHandler) CreateConnection(ctx context.Context, r *http.Request } func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*blobberhttp.CommitResult, error) { - var prevChainHash string startTime := time.Now() if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use POST instead") @@ -566,74 +565,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b "Max size reached for the allocation with this blobber") } - writeMarkerString := r.FormValue("write_marker") - if writeMarkerString == "" { - return nil, common.NewError("invalid_parameters", "Invalid write marker passed") - } - writeMarker := writemarker.WriteMarker{} - err = json.Unmarshal([]byte(writeMarkerString), &writeMarker) - if err != nil { - return nil, common.NewErrorf("invalid_parameters", - "Invalid parameters. Error parsing the writemarker for commit: %v", - err) - } - var result blobberhttp.CommitResult - var latestWriteMarkerEntity *writemarker.WriteMarkerEntity - if allocationObj.AllocationRoot == "" { - latestWriteMarkerEntity = nil - } else { - latestWriteMarkerEntity, err = writemarker.GetWriteMarkerEntity(ctx, - allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewErrorf("latest_write_marker_read_error", - "Error reading the latest write marker for allocation: %v", err) - } - if latestWriteMarkerEntity.Status == writemarker.Failed { - return nil, common.NewError("latest_write_marker_failed", - "Latest write marker is in failed state") - } - - if latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size != writeMarker.ChainSize { - return nil, common.NewErrorf("invalid_chain_size", - "Invalid chain size. expected:%v got %v", latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size, writeMarker.ChainSize) - } - - if latestWriteMarkerEntity.Status != writemarker.Committed { - writeMarker.ChainLength = latestWriteMarkerEntity.WM.ChainLength - } - prevChainHash = latestWriteMarkerEntity.WM.ChainHash - } - - writemarkerEntity := &writemarker.WriteMarkerEntity{} - writemarkerEntity.WM = writeMarker - writemarkerEntity.WM.ChainLength += 1 - if writemarkerEntity.WM.ChainLength > config.Configuration.MaxChainLength { - return nil, common.NewError("chain_length_exceeded", "Chain length exceeded") - } - - err = writemarkerEntity.VerifyMarker(ctx, allocationObj, connectionObj, latestWriteMarkerEntity) - if err != nil { - result.AllocationRoot = allocationObj.AllocationRoot - result.ErrorMessage = "Verification of write marker failed: " + err.Error() - result.Success = false - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity - } - Logger.Error("verify_writemarker_failed", zap.Error(err)) - return &result, common.NewError("write_marker_verification_failed", result.ErrorMessage) - } - - elapsedVerifyWM := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - - var clientIDForWriteRedeem = writeMarker.ClientID - - if err := writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil { - return nil, err - } - - elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - elapsedGetConnObj - elapsedVerifyWM fileIDMetaStr := r.FormValue("file_id_meta") if fileIDMetaStr == "" { @@ -652,74 +584,28 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("move_to_filestore_error", fmt.Sprintf("Error while moving to filestore: %s", err.Error())) } - elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem + elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - rootRef, err := connectionObj.ApplyChanges( - ctx, writeMarker.AllocationRoot, writeMarker.PreviousAllocationRoot, writeMarker.Timestamp, fileIDMeta) + err = connectionObj.ApplyChanges( + ctx, common.Now(), fileIDMeta) if err != nil { Logger.Error("Error applying changes", zap.Error(err)) return nil, err } - if !rootRef.IsPrecommit { - return nil, common.NewError("no_root_change", "No change in root ref") - } elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - - allocationRoot := rootRef.Hash - fileMetaRoot := rootRef.FileMetaHash - if allocationRoot != writeMarker.AllocationRoot { - result.AllocationRoot = allocationObj.AllocationRoot - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity - } - result.Success = false - result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root." + - " Expected hash: " + allocationRoot - return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage) - } - - chainHash := writemarker.CalculateChainHash(prevChainHash, allocationRoot) - if chainHash != writeMarker.ChainHash { - return nil, common.NewError("chain_hash_mismatch", "Chain hash in the write marker does not match the calculated chain hash") - } - - if fileMetaRoot != writeMarker.FileMetaRoot { - // result.AllocationRoot = allocationObj.AllocationRoot - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity - } - result.Success = false - result.ErrorMessage = "File meta root in the write marker does not match the calculated file meta root." + - " Expected hash: " + fileMetaRoot + "; Got: " + writeMarker.FileMetaRoot - return &result, common.NewError("file_meta_root_mismatch", result.ErrorMessage) - } - - writemarkerEntity.ConnectionID = connectionObj.ID - writemarkerEntity.ClientPublicKey = clientKey + elapsedGetConnObj db := datastore.GetStore().GetTransaction(ctx) - writemarkerEntity.Latest = true - if err = db.Create(writemarkerEntity).Error; err != nil { - return nil, common.NewError("write_marker_error", "Error persisting the write marker") - } - allocationObj.AllocationRoot = allocationRoot - allocationObj.FileMetaRoot = fileMetaRoot - allocationObj.IsRedeemRequired = true allocationObj.BlobberSizeUsed += connectionObj.Size allocationObj.UsedSize += connectionObj.Size updateMap := map[string]interface{}{ - "allocation_root": allocationRoot, - "file_meta_root": fileMetaRoot, "used_size": allocationObj.UsedSize, "blobber_size_used": allocationObj.BlobberSizeUsed, "is_redeem_required": true, } updateOption := func(a *allocation.Allocation) { - a.AllocationRoot = allocationRoot - a.FileMetaRoot = fileMetaRoot a.IsRedeemRequired = true a.BlobberSizeUsed = allocationObj.BlobberSizeUsed a.UsedSize = allocationObj.UsedSize @@ -730,7 +616,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } elapsedSaveAllocation := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - elapsedApplyChanges + elapsedGetConnObj - elapsedApplyChanges err = connectionObj.CommitToFileStore(ctx) if err != nil { @@ -738,13 +624,12 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("file_store_error", "Error committing to file store. "+err.Error()) } } - elapsedCommitStore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - elapsedApplyChanges - elapsedSaveAllocation - logging.Logger.Info("commit_filestore", zap.String("allocation_id", allocationId), zap.String("allocation_root", allocationRoot)) + elapsedCommitStore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedApplyChanges - elapsedSaveAllocation + logging.Logger.Info("commit_filestore", zap.String("allocation_id", allocationId)) connectionObj.DeleteChanges(ctx) db.Model(connectionObj).Updates(allocation.AllocationChangeCollector{Status: allocation.CommittedConnection}) result.AllocationRoot = allocationObj.AllocationRoot - result.WriteMarker = writemarkerEntity result.Success = true result.ErrorMessage = "" commitOperation := connectionObj.Changes[0].Operation @@ -758,13 +643,10 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b Logger.Info("[commit]"+commitOperation, zap.String("alloc_id", allocationID), - zap.String("allocation_root", writeMarker.AllocationRoot), zap.String("input", input), zap.Duration("get_alloc", elapsedAllocation), zap.Duration("get-lock", elapsedGetLock), zap.Duration("get-conn-obj", elapsedGetConnObj), - zap.Duration("verify-wm", elapsedVerifyWM), - zap.Duration("write-pre-redeem", elapsedWritePreRedeem), zap.Duration("move-to-filestore", elapsedMoveToFilestore), zap.Duration("apply-changes", elapsedApplyChanges), zap.Duration("save-allocation", elapsedSaveAllocation), @@ -775,313 +657,313 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (interface{}, error) { - allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) - allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) - allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - if !allocationObj.CanRename() { - return nil, common.NewError("prohibited_allocation_file_options", "Cannot rename data in this allocation.") - } - - allocationID := allocationObj.ID - - clientID := ctx.Value(constants.ContextKeyClient).(string) - _ = ctx.Value(constants.ContextKeyClientKey).(string) - valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - if clientID == "" { - return nil, common.NewError("invalid_operation", "Invalid client") - } - - new_name := r.FormValue("new_name") - if new_name == "" { - return nil, common.NewError("invalid_parameters", "Invalid name") - } - - pathHash, err := pathHashFromReq(r, allocationID) - if err != nil { - return nil, err - } - - if clientID == "" || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - connectionID := r.FormValue("connection_id") - if connectionID == "" { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - - connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - if err != nil { - return nil, common.NewError("meta_error", "Error reading metadata for connection") - } - - objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root", "type"}) - - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - - if objectRef.Path == "/" { - return nil, common.NewError("invalid_operation", "cannot rename root path") - } - - allocationChange := &allocation.AllocationChange{} - allocationChange.ConnectionID = connectionObj.ID - allocationChange.Size = 0 - allocationChange.LookupHash = pathHash - allocationChange.Operation = constants.FileOperationRename - dfc := &allocation.RenameFileChange{ConnectionID: connectionObj.ID, - AllocationID: connectionObj.AllocationID, Path: objectRef.Path, Type: objectRef.Type} - dfc.NewName = new_name - connectionObj.AddChange(allocationChange, dfc) - - err = connectionObj.Save(ctx) - if err != nil { - Logger.Error("Error in writing the connection meta data", zap.Error(err)) - return nil, common.NewError("connection_write_error", "Error writing the connection meta data") - } - - result := &allocation.UploadResult{} - result.Filename = new_name - result.Hash = objectRef.Hash - result.ValidationRoot = objectRef.ValidationRoot - result.FixedMerkleRoot = objectRef.FixedMerkleRoot - result.Size = objectRef.Size - - return result, nil + // allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + // allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) + // allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + // if err != nil { + // return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + // } + + // if !allocationObj.CanRename() { + // return nil, common.NewError("prohibited_allocation_file_options", "Cannot rename data in this allocation.") + // } + + // allocationID := allocationObj.ID + + // clientID := ctx.Value(constants.ContextKeyClient).(string) + // _ = ctx.Value(constants.ContextKeyClientKey).(string) + // valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) + // if !valid || err != nil { + // return nil, common.NewError("invalid_signature", "Invalid signature") + // } + + // if clientID == "" { + // return nil, common.NewError("invalid_operation", "Invalid client") + // } + + // new_name := r.FormValue("new_name") + // if new_name == "" { + // return nil, common.NewError("invalid_parameters", "Invalid name") + // } + + // pathHash, err := pathHashFromReq(r, allocationID) + // if err != nil { + // return nil, err + // } + + // if clientID == "" || allocationObj.OwnerID != clientID { + // return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + // } + + // connectionID := r.FormValue("connection_id") + // if connectionID == "" { + // return nil, common.NewError("invalid_parameters", "Invalid connection id passed") + // } + + // connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) + // if err != nil { + // return nil, common.NewError("meta_error", "Error reading metadata for connection") + // } + + // objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root", "type"}) + + // if err != nil { + // return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + // } + + // if objectRef.Path == "/" { + // return nil, common.NewError("invalid_operation", "cannot rename root path") + // } + + // allocationChange := &allocation.AllocationChange{} + // allocationChange.ConnectionID = connectionObj.ID + // allocationChange.Size = 0 + // allocationChange.LookupHash = pathHash + // allocationChange.Operation = constants.FileOperationRename + // dfc := &allocation.RenameFileChange{ConnectionID: connectionObj.ID, + // AllocationID: connectionObj.AllocationID, Path: objectRef.Path, Type: objectRef.Type} + // dfc.NewName = new_name + // connectionObj.AddChange(allocationChange, dfc) + + // err = connectionObj.Save(ctx) + // if err != nil { + // Logger.Error("Error in writing the connection meta data", zap.Error(err)) + // return nil, common.NewError("connection_write_error", "Error writing the connection meta data") + // } + + // result := &allocation.UploadResult{} + // result.Filename = new_name + // result.Hash = objectRef.Hash + // result.ValidationRoot = objectRef.ValidationRoot + // result.FixedMerkleRoot = objectRef.FixedMerkleRoot + // result.Size = objectRef.Size + + return nil, errors.New("not implemented") } func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (interface{}, error) { - allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) - allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) - allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - if !allocationObj.CanCopy() { - return nil, common.NewError("prohibited_allocation_file_options", "Cannot copy data from this allocation.") - } - - valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - clientID := ctx.Value(constants.ContextKeyClient).(string) - _ = ctx.Value(constants.ContextKeyClientKey).(string) - - allocationID := allocationObj.ID - - if clientID == "" { - return nil, common.NewError("invalid_operation", "Invalid client") - } - - destPath := r.FormValue("dest") - if destPath == "" { - return nil, common.NewError("invalid_parameters", "Invalid destination for operation") - } - - pathHash, err := pathHashFromReq(r, allocationID) - if err != nil { - return nil, err - } - - if clientID == "" || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - connectionID := r.FormValue("connection_id") - if connectionID == "" { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - - connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - if err != nil { - return nil, common.NewError("meta_error", "Error reading metadata for connection") - } - - objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) - - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - if objectRef.ParentPath == destPath || objectRef.Path == destPath { - return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot copy to the same parent directory.") - } - newPath := filepath.Join(destPath, objectRef.Name) - paths, err := common.GetParentPaths(newPath) - if err != nil { - return nil, err - } - - paths = append(paths, newPath) - - refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) - if err != nil { - Logger.Error("Database error", zap.Error(err)) - return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) - } - - for _, ref := range refs { - switch ref.Path { - case newPath: - return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") - default: - if ref.Type == reference.FILE { - return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) - } - } - } - - allocationChange := &allocation.AllocationChange{} - allocationChange.ConnectionID = connectionObj.ID - allocationChange.Size = objectRef.Size - allocationChange.LookupHash = pathHash - allocationChange.Operation = constants.FileOperationCopy - dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ID, - AllocationID: connectionObj.AllocationID, DestPath: destPath} - dfc.SrcPath = objectRef.Path - allocation.UpdateConnectionObjSize(connectionID, allocationChange.Size) - connectionObj.AddChange(allocationChange, dfc) - - err = connectionObj.Save(ctx) - if err != nil { - Logger.Error("Error in writing the connection meta data", zap.Error(err)) - return nil, common.NewError("connection_write_error", "Error writing the connection meta data") - } - - result := &allocation.UploadResult{} - result.Filename = objectRef.Name - result.Hash = objectRef.Hash - result.ValidationRoot = objectRef.ValidationRoot - result.FixedMerkleRoot = objectRef.FixedMerkleRoot - result.Size = objectRef.Size - return result, nil + // allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + // allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) + // allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + // if err != nil { + // return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + // } + + // if !allocationObj.CanCopy() { + // return nil, common.NewError("prohibited_allocation_file_options", "Cannot copy data from this allocation.") + // } + + // valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) + // if !valid || err != nil { + // return nil, common.NewError("invalid_signature", "Invalid signature") + // } + + // clientID := ctx.Value(constants.ContextKeyClient).(string) + // _ = ctx.Value(constants.ContextKeyClientKey).(string) + + // allocationID := allocationObj.ID + + // if clientID == "" { + // return nil, common.NewError("invalid_operation", "Invalid client") + // } + + // destPath := r.FormValue("dest") + // if destPath == "" { + // return nil, common.NewError("invalid_parameters", "Invalid destination for operation") + // } + + // pathHash, err := pathHashFromReq(r, allocationID) + // if err != nil { + // return nil, err + // } + + // if clientID == "" || allocationObj.OwnerID != clientID { + // return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + // } + + // connectionID := r.FormValue("connection_id") + // if connectionID == "" { + // return nil, common.NewError("invalid_parameters", "Invalid connection id passed") + // } + + // connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) + // if err != nil { + // return nil, common.NewError("meta_error", "Error reading metadata for connection") + // } + + // objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) + + // if err != nil { + // return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + // } + // if objectRef.ParentPath == destPath || objectRef.Path == destPath { + // return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot copy to the same parent directory.") + // } + // newPath := filepath.Join(destPath, objectRef.Name) + // paths, err := common.GetParentPaths(newPath) + // if err != nil { + // return nil, err + // } + + // paths = append(paths, newPath) + + // refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) + // if err != nil { + // Logger.Error("Database error", zap.Error(err)) + // return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) + // } + + // for _, ref := range refs { + // switch ref.Path { + // case newPath: + // return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") + // default: + // if ref.Type == reference.FILE { + // return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) + // } + // } + // } + + // allocationChange := &allocation.AllocationChange{} + // allocationChange.ConnectionID = connectionObj.ID + // allocationChange.Size = objectRef.Size + // allocationChange.LookupHash = pathHash + // allocationChange.Operation = constants.FileOperationCopy + // dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ID, + // AllocationID: connectionObj.AllocationID, DestPath: destPath} + // dfc.SrcPath = objectRef.Path + // allocation.UpdateConnectionObjSize(connectionID, allocationChange.Size) + // connectionObj.AddChange(allocationChange, dfc) + + // err = connectionObj.Save(ctx) + // if err != nil { + // Logger.Error("Error in writing the connection meta data", zap.Error(err)) + // return nil, common.NewError("connection_write_error", "Error writing the connection meta data") + // } + + // result := &allocation.UploadResult{} + // result.Filename = objectRef.Name + // result.Hash = objectRef.Hash + // result.ValidationRoot = objectRef.ValidationRoot + // result.FixedMerkleRoot = objectRef.FixedMerkleRoot + // result.Size = objectRef.Size + return nil, errors.New("not implemented") } func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (interface{}, error) { - allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) - allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) - allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - } - - if !allocationObj.CanMove() { - return nil, common.NewError("prohibited_allocation_file_options", "Cannot move data in this allocation.") - } - - valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) - if !valid || err != nil { - return nil, common.NewError("invalid_signature", "Invalid signature") - } - - clientID := ctx.Value(constants.ContextKeyClient).(string) - _ = ctx.Value(constants.ContextKeyClientKey).(string) - - allocationID := allocationObj.ID - - if clientID == "" { - return nil, common.NewError("invalid_operation", "Invalid client") - } - - destPath := r.FormValue("dest") - if destPath == "" { - return nil, common.NewError("invalid_parameters", "Invalid destination for operation") - } - - pathHash, err := pathHashFromReq(r, allocationID) - if err != nil { - return nil, err - } - - if clientID == "" || allocationObj.OwnerID != clientID { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - } - - connectionID := r.FormValue("connection_id") - if connectionID == "" { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - - connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - if err != nil { - return nil, common.NewError("meta_error", "Error reading metadata for connection") - } - - objectRef, err := reference.GetLimitedRefFieldsByLookupHash( - ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) - - if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - } - - if objectRef.ParentPath == destPath { - return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot move to the same parent directory.") - } - newPath := filepath.Join(destPath, objectRef.Name) - paths, err := common.GetParentPaths(newPath) - if err != nil { - return nil, err - } - - paths = append(paths, newPath) - - refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) - if err != nil { - Logger.Error("Database error", zap.Error(err)) - return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) - } - - for _, ref := range refs { - switch ref.Path { - case newPath: - return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") - default: - if ref.Type == reference.FILE { - return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) - } - } - } - - allocationChange := &allocation.AllocationChange{} - allocationChange.ConnectionID = connectionObj.ID - allocationChange.Size = 0 - allocationChange.LookupHash = pathHash - allocationChange.Operation = constants.FileOperationMove - dfc := &allocation.MoveFileChange{ - ConnectionID: connectionObj.ID, - AllocationID: connectionObj.AllocationID, - SrcPath: objectRef.Path, - DestPath: destPath, - } - dfc.SrcPath = objectRef.Path - connectionObj.AddChange(allocationChange, dfc) - - err = connectionObj.Save(ctx) - if err != nil { - Logger.Error("Error in writing the connection meta data", zap.Error(err)) - return nil, common.NewError("connection_write_error", "Error writing the connection meta data") - } - - result := &allocation.UploadResult{} - result.Filename = objectRef.Name - result.Hash = objectRef.Hash - result.ValidationRoot = objectRef.ValidationRoot - result.FixedMerkleRoot = objectRef.FixedMerkleRoot - result.Size = objectRef.Size - return result, nil + // allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) + // allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + // allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + // if err != nil { + // return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + // } + + // if !allocationObj.CanMove() { + // return nil, common.NewError("prohibited_allocation_file_options", "Cannot move data in this allocation.") + // } + + // valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) + // if !valid || err != nil { + // return nil, common.NewError("invalid_signature", "Invalid signature") + // } + + // clientID := ctx.Value(constants.ContextKeyClient).(string) + // _ = ctx.Value(constants.ContextKeyClientKey).(string) + + // allocationID := allocationObj.ID + + // if clientID == "" { + // return nil, common.NewError("invalid_operation", "Invalid client") + // } + + // destPath := r.FormValue("dest") + // if destPath == "" { + // return nil, common.NewError("invalid_parameters", "Invalid destination for operation") + // } + + // pathHash, err := pathHashFromReq(r, allocationID) + // if err != nil { + // return nil, err + // } + + // if clientID == "" || allocationObj.OwnerID != clientID { + // return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + // } + + // connectionID := r.FormValue("connection_id") + // if connectionID == "" { + // return nil, common.NewError("invalid_parameters", "Invalid connection id passed") + // } + + // connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) + // if err != nil { + // return nil, common.NewError("meta_error", "Error reading metadata for connection") + // } + + // objectRef, err := reference.GetLimitedRefFieldsByLookupHash( + // ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) + + // if err != nil { + // return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + // } + + // if objectRef.ParentPath == destPath { + // return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot move to the same parent directory.") + // } + // newPath := filepath.Join(destPath, objectRef.Name) + // paths, err := common.GetParentPaths(newPath) + // if err != nil { + // return nil, err + // } + + // paths = append(paths, newPath) + + // refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) + // if err != nil { + // Logger.Error("Database error", zap.Error(err)) + // return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) + // } + + // for _, ref := range refs { + // switch ref.Path { + // case newPath: + // return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") + // default: + // if ref.Type == reference.FILE { + // return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) + // } + // } + // } + + // allocationChange := &allocation.AllocationChange{} + // allocationChange.ConnectionID = connectionObj.ID + // allocationChange.Size = 0 + // allocationChange.LookupHash = pathHash + // allocationChange.Operation = constants.FileOperationMove + // dfc := &allocation.MoveFileChange{ + // ConnectionID: connectionObj.ID, + // AllocationID: connectionObj.AllocationID, + // SrcPath: objectRef.Path, + // DestPath: destPath, + // } + // dfc.SrcPath = objectRef.Path + // connectionObj.AddChange(allocationChange, dfc) + + // err = connectionObj.Save(ctx) + // if err != nil { + // Logger.Error("Error in writing the connection meta data", zap.Error(err)) + // return nil, common.NewError("connection_write_error", "Error writing the connection meta data") + // } + + // result := &allocation.UploadResult{} + // result.Filename = objectRef.Name + // result.Hash = objectRef.Hash + // result.ValidationRoot = objectRef.ValidationRoot + // result.FixedMerkleRoot = objectRef.FixedMerkleRoot + // result.Size = objectRef.Size + return nil, errors.New("not implemented") } func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, connectionObj *allocation.AllocationChangeCollector) (*allocation.UploadResult, error) { diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index b5881b4bb..19cf49c30 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -39,6 +39,10 @@ func cleanupAllocationFiles(ctx context.Context, allocationObj allocation.Alloca db := datastore.GetStore().GetTransaction(ctx) _ = filestore.GetFileStore().IterateObjects(allocationObj.ID, func(hash string, contentSize int64) { + // thumbnail suffix makes hash greater than 65 + if len(hash) > 65 { + return + } var refs []reference.Ref version := 0 if len(hash) > 64 { @@ -46,8 +50,7 @@ func cleanupAllocationFiles(ctx context.Context, allocationObj allocation.Alloca hash = hash[:64] } err := db.Table((reference.Ref{}).TableName()). - Where(reference.Ref{ValidationRoot: hash, Type: reference.FILE}). - Or(reference.Ref{ThumbnailHash: hash, Type: reference.FILE}). + Where(reference.Ref{LookupHash: hash, Type: reference.FILE}). Find(&refs).Error if err != nil { diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 31d2f0f87..aad965182 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -2,13 +2,14 @@ package reference import ( "context" - "path/filepath" + "database/sql" + "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -func DeleteObject(ctx context.Context, rootRef *Ref, allocationID, objPath string, ts common.Timestamp) error { +func DeleteObject(ctx context.Context, allocationID, objPath string, ts common.Timestamp) error { likePath := objPath + "/%" if objPath == "/" { likePath = "/%" @@ -16,60 +17,12 @@ func DeleteObject(ctx context.Context, rootRef *Ref, allocationID, objPath strin db := datastore.GetStore().GetTransaction(ctx) - err := db.Exec("UPDATE reference_objects SET is_precommit=? WHERE allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", true, allocationID, "/", objPath, likePath).Error + err := db.Exec("UPDATE reference_objects SET is_precommit=?,deleted_at=? WHERE allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", true, sql.NullTime{ + Time: time.Now(), + Valid: true, + }, allocationID, "/", objPath, likePath).Error if err != nil { return err } - - err = db.Delete(&Ref{}, "allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", - allocationID, "/", objPath, likePath).Error - - if err != nil { - return err - } - if objPath == "/" { - rootRef.Children = nil - rootRef.HashToBeComputed = true - rootRef.childrenLoaded = true - rootRef.UpdatedAt = ts - return nil - } - parentPath, deleteFileName := filepath.Split(objPath) - rootRef.UpdatedAt = ts - fields, err := common.GetPathFields(parentPath) - if err != nil { - return err - } - - dirRef := rootRef - - for _, name := range fields { - var found bool - for _, ref := range dirRef.Children { - if ref.Name == name { - ref.HashToBeComputed = true - ref.childrenLoaded = true - ref.UpdatedAt = ts - found = true - dirRef = ref - break - } - } - - if !found { - return common.NewError("invalid_reference_path", "Reference path has invalid references") - } - } - - for i, child := range dirRef.Children { - basePath := filepath.Base(child.Path) - if basePath == deleteFileName || child.Path == objPath { - dirRef.RemoveChild(i) - break - } - } - - rootRef.HashToBeComputed = true - rootRef.childrenLoaded = true - return nil + return err } diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index e5d6ae223..f46f04884 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -308,6 +308,22 @@ func IsRefExist(ctx context.Context, allocationID, path string) (bool, error) { return Found, nil } +func GetObjectSizeByLookupHash(ctx context.Context, lookupHash string) (int64, error) { + db := datastore.GetStore().GetTransaction(ctx) + var size int64 + err := db.Model(&Ref{}). + Select("size"). + Where("lookup_hash = ?", lookupHash). + Take(&size).Error + if err != nil { + if err == gorm.ErrRecordNotFound { + return 0, nil + } + return 0, err + } + return size, nil +} + // GetRefsTypeFromPaths Give list of paths it will return refs of respective path with only Type and Path selected in sql query func GetRefsTypeFromPaths(ctx context.Context, allocationID string, paths []string) (refs []*Ref, err error) { if len(paths) == 0 { From b34a7e8b315e95d4f5aff37528ff23d5039f93da Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 11 Jul 2024 00:18:59 +0530 Subject: [PATCH 04/98] remove unused columns in ref table --- .../allocation/file_changer_update.go | 1 - .../0chain.net/blobbercore/convert/convert.go | 8 -------- .../blobbercore/handler/file_command_delete.go | 2 -- .../handler/object_operation_handler.go | 18 +++--------------- .../blobbercore/handler/storage_handler.go | 2 +- .../go/0chain.net/blobbercore/reference/ref.go | 15 +-------------- 6 files changed, 5 insertions(+), 41 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 3b94d3352..be91ffc8e 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -82,7 +82,6 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.DataHash = nf.DataHash fileRef.DataHashSignature = nf.DataHashSignature fileRef.LookupHash = nf.LookupHash - fileRef.AllocationRoot = allocationRoot fileRef.Size = nf.Size fileRef.ThumbnailHash = nf.ThumbnailHash fileRef.ThumbnailSize = nf.ThumbnailSize diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index 8f3ca5c41..1c7946c05 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -282,11 +282,8 @@ func convertFileRefToFileMetaDataGRPC(fileref *reference.Ref) *blobbergrpc.FileM Path: fileref.Path, Hash: fileref.Hash, NumBlocks: fileref.NumBlocks, - PathHash: fileref.PathHash, CustomMeta: fileref.CustomMeta, - ValidationRoot: fileref.ValidationRoot, Size: fileref.Size, - FixedMerkleRoot: fileref.FixedMerkleRoot, ActualFileSize: fileref.ActualFileSize, ActualFileHash: fileref.ActualFileHash, MimeType: fileref.MimeType, @@ -308,7 +305,6 @@ func convertDirRefToDirMetaDataGRPC(dirref *reference.Ref) *blobbergrpc.DirMetaD Path: dirref.Path, Hash: dirref.Hash, NumBlocks: dirref.NumBlocks, - PathHash: dirref.PathHash, Size: dirref.Size, CreatedAt: int64(dirref.CreatedAt), UpdatedAt: int64(dirref.UpdatedAt), @@ -338,11 +334,8 @@ func convertFileMetaDataGRPCToFileRef(metaData *blobbergrpc.FileMetaData) *refer Path: metaData.Path, Hash: metaData.Hash, NumBlocks: metaData.NumBlocks, - PathHash: metaData.PathHash, CustomMeta: metaData.CustomMeta, - ValidationRoot: metaData.ValidationRoot, Size: metaData.Size, - FixedMerkleRoot: metaData.FixedMerkleRoot, ActualFileSize: metaData.ActualFileSize, ActualFileHash: metaData.ActualFileHash, MimeType: metaData.MimeType, @@ -364,7 +357,6 @@ func convertDirMetaDataGRPCToDirRef(dirref *blobbergrpc.DirMetaData) *reference. Path: dirref.Path, Hash: dirref.Hash, NumBlocks: dirref.NumBlocks, - PathHash: dirref.PathHash, Size: dirref.Size, CreatedAt: common.Timestamp(dirref.CreatedAt), UpdatedAt: common.Timestamp(dirref.UpdatedAt), diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 38497b283..972a04abd 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -86,8 +86,6 @@ func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *a result := allocation.UploadResult{} result.Filename = cmd.existingFileRef.Name - result.ValidationRoot = cmd.existingFileRef.ValidationRoot - result.FixedMerkleRoot = cmd.existingFileRef.FixedMerkleRoot result.Size = cmd.existingFileRef.Size result.UpdateChange = true diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index a57fbc0fa..439ecdb06 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -373,17 +373,12 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, common.NewErrorf("download_file", "BlockNum or NumBlocks is too large to convert to int") } - fromPreCommit := false + fromPreCommit := fileref.IsPrecommit if downloadMode == DownloadContentThumb { - - if fileref.IsPrecommit { - fromPreCommit = fileref.ThumbnailHash != fileref.PrevThumbnailHash - } - rbi := &filestore.ReadBlockInput{ AllocationID: alloc.ID, FileSize: fileref.ThumbnailSize, - Hash: fileref.ThumbnailHash, + Hash: fileref.LookupHash, StartBlockNum: int(dr.BlockNum), NumBlocks: int(dr.NumBlocks), IsThumbnail: true, @@ -397,15 +392,10 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, common.NewErrorf("download_file", "couldn't get thumbnail block: %v", err) } } else { - - if fileref.IsPrecommit { - fromPreCommit = fileref.ValidationRoot != fileref.PrevValidationRoot - } - rbi := &filestore.ReadBlockInput{ AllocationID: alloc.ID, FileSize: fileref.Size, - Hash: fileref.ValidationRoot, + Hash: fileref.LookupHash, StartBlockNum: int(dr.BlockNum), NumBlocks: int(dr.NumBlocks), VerifyDownload: dr.VerifyDownload, @@ -997,8 +987,6 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn result := &allocation.UploadResult{} result.Filename = fileRef.Name result.Hash = fileRef.Hash - result.ValidationRoot = fileRef.ValidationRoot - result.FixedMerkleRoot = fileRef.FixedMerkleRoot result.Size = fileRef.Size return result, nil diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 2f99acbbf..11c4d73a6 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -255,7 +255,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i if err != nil { return nil, common.NewError("bad_db_operation", "Error retrieving file stats. "+err.Error()) } - wm, _ := writemarker.GetWriteMarkerEntity(ctx, fileref.AllocationRoot) + wm, _ := writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) if wm != nil && fileStats != nil { fileStats.WriteMarkerRedeemTxn = wm.CloseTxnID fileStats.OnChain = wm.OnChain() diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index f46f04884..54030eabb 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -57,23 +57,16 @@ type Ref struct { FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` Hash string `gorm:"column:hash;size:64;not null" dirlist:"hash" filelist:"hash"` NumBlocks int64 `gorm:"column:num_of_blocks;not null;default:0" dirlist:"num_of_blocks" filelist:"num_of_blocks"` - PathHash string `gorm:"column:path_hash;size:64;not null" dirlist:"path_hash" filelist:"path_hash"` ParentPath string `gorm:"column:parent_path;size:999;index:idx_parent_path_alloc,priority:2"` PathLevel int `gorm:"column:level;not null;default:0"` CustomMeta string `gorm:"column:custom_meta;not null" filelist:"custom_meta"` - ValidationRoot string `gorm:"column:validation_root;size:64;not null;index:idx_validation_alloc,priority:2" filelist:"validation_root"` - PrevValidationRoot string `gorm:"column:prev_validation_root" filelist:"prev_validation_root" json:"prev_validation_root"` - ValidationRootSignature string `gorm:"column:validation_root_signature;size:64" filelist:"validation_root_signature" json:"validation_root_signature,omitempty"` Size int64 `gorm:"column:size;not null;default:0" dirlist:"size" filelist:"size"` - FixedMerkleRoot string `gorm:"column:fixed_merkle_root;size:64;not null" filelist:"fixed_merkle_root"` ActualFileSize int64 `gorm:"column:actual_file_size;not null;default:0" dirlist:"actual_file_size" filelist:"actual_file_size"` ActualFileHashSignature string `gorm:"column:actual_file_hash_signature;size:64" filelist:"actual_file_hash_signature" json:"actual_file_hash_signature,omitempty"` ActualFileHash string `gorm:"column:actual_file_hash;size:64;not null" filelist:"actual_file_hash"` MimeType string `gorm:"column:mimetype;size:255;not null" filelist:"mimetype"` - AllocationRoot string `gorm:"column:allocation_root;size:64;not null"` ThumbnailSize int64 `gorm:"column:thumbnail_size;not null;default:0" filelist:"thumbnail_size"` ThumbnailHash string `gorm:"column:thumbnail_hash;size:64;not null" filelist:"thumbnail_hash"` - PrevThumbnailHash string `gorm:"column:prev_thumbnail_hash" filelist:"prev_thumbnail_hash"` ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size;not null;default:0" filelist:"actual_thumbnail_size"` ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash;size:64;not null" filelist:"actual_thumbnail_hash"` EncryptedKey string `gorm:"column:encrypted_key;size:64" filelist:"encrypted_key"` @@ -432,14 +425,12 @@ func (r *Ref) GetFileMetaHashData() string { func (fr *Ref) GetFileHashData() string { return fmt.Sprintf( - "%s:%s:%s:%s:%d:%s:%s:%d:%s:%d:%s", + "%s:%s:%s:%s:%d:%d:%s:%d:%s", fr.AllocationID, fr.Type, // don't need to add it as well fr.Name, // don't see any utility as fr.Path below has name in it fr.Path, fr.Size, - fr.ValidationRoot, - fr.FixedMerkleRoot, fr.ActualFileSize, fr.ActualFileHash, fr.ChunkSize, @@ -457,7 +448,6 @@ func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool, collector Q fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize))) fr.PathLevel = len(strings.Split(strings.TrimRight(fr.Path, "/"), "/")) fr.LookupHash = GetReferenceLookup(fr.AllocationID, fr.Path) - fr.PathHash = fr.LookupHash var err error if saveToDB && fr.HashToBeComputed { @@ -483,7 +473,6 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que childHashes := make([]string, l) childFileMetaHashes := make([]string, l) - childPathHashes := make([]string, l) var refNumBlocks, size, actualSize int64 for i, childRef := range r.Children { @@ -496,7 +485,6 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que childFileMetaHashes[i] = childRef.FileMetaHash childHashes[i] = childRef.Hash - childPathHashes[i] = childRef.PathHash refNumBlocks += childRef.NumBlocks size += childRef.Size actualSize += childRef.ActualFileSize @@ -504,7 +492,6 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que r.FileMetaHash = encryption.Hash(r.Path + strings.Join(childFileMetaHashes, ":")) r.Hash = encryption.Hash(r.GetHashData() + strings.Join(childHashes, ":")) - r.PathHash = encryption.Hash(strings.Join(childPathHashes, ":")) r.NumBlocks = refNumBlocks r.Size = size r.ActualFileSize = actualSize From d273658bb0b852e9a3fa79f44cc2446e074e37d2 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 11 Jul 2024 12:12:46 +0530 Subject: [PATCH 05/98] remove hash field from ref --- .../blobbercore/challenge/protocol.go | 3 --- .../0chain.net/blobbercore/convert/convert.go | 4 ---- .../handler/file_command_delete.go | 2 +- .../handler/object_operation_handler.go | 24 ++----------------- .../blobbercore/handler/storage_handler.go | 3 ++- .../blobbercore/reference/objectpath.go | 5 ++-- .../0chain.net/blobbercore/reference/ref.go | 17 ++++--------- 7 files changed, 11 insertions(+), 47 deletions(-) diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 55b8940c0..65c0c84f6 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -109,9 +109,6 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { blockNum := int64(0) var objectPath *reference.ObjectPath if rootRef != nil { - if rootRef.Hash != allocationObj.AllocationRoot { - logging.Logger.Error("root_mismatch", zap.Any("allocation_root", allocationObj.AllocationRoot), zap.Any("latest_write_marker", wms[len(wms)-1].WM.AllocationRoot), zap.Any("root_ref_hash", rootRef.Hash)) - } if rootRef.NumBlocks > 0 { r := rand.New(rand.NewSource(cr.RandomNumber)) blockNum = r.Int63n(rootRef.NumBlocks) diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index 1c7946c05..1c6ef1bb3 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -280,7 +280,6 @@ func convertFileRefToFileMetaDataGRPC(fileref *reference.Ref) *blobbergrpc.FileM LookupHash: fileref.LookupHash, Name: fileref.Name, Path: fileref.Path, - Hash: fileref.Hash, NumBlocks: fileref.NumBlocks, CustomMeta: fileref.CustomMeta, Size: fileref.Size, @@ -303,7 +302,6 @@ func convertDirRefToDirMetaDataGRPC(dirref *reference.Ref) *blobbergrpc.DirMetaD LookupHash: dirref.LookupHash, Name: dirref.Name, Path: dirref.Path, - Hash: dirref.Hash, NumBlocks: dirref.NumBlocks, Size: dirref.Size, CreatedAt: int64(dirref.CreatedAt), @@ -332,7 +330,6 @@ func convertFileMetaDataGRPCToFileRef(metaData *blobbergrpc.FileMetaData) *refer LookupHash: metaData.LookupHash, Name: metaData.Name, Path: metaData.Path, - Hash: metaData.Hash, NumBlocks: metaData.NumBlocks, CustomMeta: metaData.CustomMeta, Size: metaData.Size, @@ -355,7 +352,6 @@ func convertDirMetaDataGRPCToDirRef(dirref *blobbergrpc.DirMetaData) *reference. LookupHash: dirref.LookupHash, Name: dirref.Name, Path: dirref.Path, - Hash: dirref.Hash, NumBlocks: dirref.NumBlocks, Size: dirref.Size, CreatedAt: common.Timestamp(dirref.CreatedAt), diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 972a04abd..5f392d938 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -82,7 +82,7 @@ func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *a connectionID := cmd.connectionID cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID, AllocationID: allocationObj.ID, Name: cmd.existingFileRef.Name, - Hash: cmd.existingFileRef.Hash, Path: cmd.existingFileRef.Path, Size: deleteSize} + Hash: cmd.existingFileRef.LookupHash, Path: cmd.existingFileRef.Path, Size: deleteSize} result := allocation.UploadResult{} result.Filename = cmd.existingFileRef.Name diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 439ecdb06..ca4b2917c 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -978,7 +978,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn allocationChange.Operation = constants.FileOperationDelete dfc := &allocation.DeleteFileChange{ConnectionID: connectionObj.ID, AllocationID: connectionObj.AllocationID, Name: fileRef.Name, - Hash: fileRef.Hash, Path: fileRef.Path, Size: deleteSize} + Hash: fileRef.LookupHash, Path: fileRef.Path, Size: deleteSize} allocation.UpdateConnectionObjSize(connectionObj.ID, allocationChange.Size) @@ -986,7 +986,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn result := &allocation.UploadResult{} result.Filename = fileRef.Name - result.Hash = fileRef.Hash + result.Hash = fileRef.LookupHash result.Size = fileRef.Size return result, nil @@ -1306,25 +1306,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob } Logger.Info("rollback_root_ref", zap.Any("root_ref", rootRef)) - allocationRoot := rootRef.Hash fileMetaRoot := rootRef.FileMetaHash - - if allocationRoot != writeMarker.AllocationRoot { - result.AllocationRoot = allocationObj.AllocationRoot - result.WriteMarker = latestWriteMarkerEntity - result.Success = false - result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root." + - " Expected hash: " + allocationRoot - txn.Rollback() - return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage) - } - - chainHash := writemarker.CalculateChainHash(latestWriteMarkerEntity.WM.ChainHash, allocationRoot) - if chainHash != writeMarker.ChainHash { - txn.Rollback() - return nil, common.NewError("chain_hash_mismatch", "Chain hash in the write marker does not match the calculated chain hash") - } - if fileMetaRoot != writeMarker.FileMetaRoot { if latestWriteMarkerEntity != nil { result.WriteMarker = latestWriteMarkerEntity @@ -1349,13 +1331,11 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob alloc.BlobberSizeUsed -= latestWriteMarkerEntity.WM.Size alloc.UsedSize -= latestWriteMarkerEntity.WM.Size - alloc.AllocationRoot = allocationRoot alloc.FileMetaRoot = fileMetaRoot alloc.IsRedeemRequired = true updateMap := map[string]interface{}{ "blobber_size_used": alloc.BlobberSizeUsed, "used_size": alloc.UsedSize, - "allocation_root": alloc.AllocationRoot, "file_meta_root": alloc.FileMetaRoot, "is_redeem_required": true, } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 11c4d73a6..2cf444e92 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -554,7 +554,8 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request if allocationObj.AllocationRoot == "" { latestWM = nil } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, rootRef.Hash) + //TODO: remove latestWM + latestWM, err = writemarker.GetWriteMarkerEntity(ctx, rootRef.FileMetaHash) if err != nil { errCh <- common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) return diff --git a/code/go/0chain.net/blobbercore/reference/objectpath.go b/code/go/0chain.net/blobbercore/reference/objectpath.go index d9d19fcb3..34d817b35 100644 --- a/code/go/0chain.net/blobbercore/reference/objectpath.go +++ b/code/go/0chain.net/blobbercore/reference/objectpath.go @@ -31,7 +31,7 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O if rootRef.NumBlocks == 0 { var retObj ObjectPath - retObj.RootHash = rootRef.Hash + // retObj.RootHash = rootRef.Hash retObj.FileBlockNum = 0 result := rootRef.GetListingData(ctx) list := make([]map[string]interface{}, len(rootRef.Children)) @@ -68,7 +68,7 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O break } curRef, err = GetRefWithSortedChildren(ctx, allocationID, child.Path) - if err != nil || curRef.Hash == "" { + if err != nil { return nil, common.NewError("failed_object_path", "Failed to get the object path") } curResult = list[idx] @@ -80,7 +80,6 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O } var retObj ObjectPath - retObj.RootHash = rootRef.Hash retObj.Meta = curRef.GetListingData(ctx) retObj.Path = result retObj.FileBlockNum = remainingBlocks diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 54030eabb..516175618 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -55,7 +55,6 @@ type Ref struct { Name string `gorm:"column:name;size:100;not null;index:idx_name_gin" dirlist:"name" filelist:"name"` // uses GIN tsvector index for full-text search Path string `gorm:"column:path;size:1000;not null;index:idx_path_alloc,priority:2;index:path_idx;index:idx_path_gin_trgm" dirlist:"path" filelist:"path"` FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` - Hash string `gorm:"column:hash;size:64;not null" dirlist:"hash" filelist:"hash"` NumBlocks int64 `gorm:"column:num_of_blocks;not null;default:0" dirlist:"num_of_blocks" filelist:"num_of_blocks"` ParentPath string `gorm:"column:parent_path;size:999;index:idx_parent_path_alloc,priority:2"` PathLevel int `gorm:"column:level;not null;default:0"` @@ -120,25 +119,21 @@ type PaginatedRef struct { //Gorm smart select fields. Path string `gorm:"column:path" json:"path,omitempty"` Hash string `gorm:"column:hash" json:"hash,omitempty"` NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_blocks,omitempty"` - PathHash string `gorm:"column:path_hash" json:"path_hash,omitempty"` ParentPath string `gorm:"column:parent_path" json:"parent_path,omitempty"` PathLevel int `gorm:"column:level" json:"level,omitempty"` CustomMeta string `gorm:"column:custom_meta" json:"custom_meta,omitempty"` - ValidationRootSignature string `gorm:"column:validation_root_signature" json:"validation_root_signature,omitempty"` - ValidationRoot string `gorm:"column:validation_root" json:"validation_root,omitempty"` Size int64 `gorm:"column:size" json:"size,omitempty"` - FixedMerkleRoot string `gorm:"column:fixed_merkle_root" json:"fixed_merkle_root,omitempty"` ActualFileSize int64 `gorm:"column:actual_file_size" json:"actual_file_size,omitempty"` ActualFileHashSignature string `gorm:"column:actual_file_hash_signature" json:"actual_file_hash_signature,omitempty"` ActualFileHash string `gorm:"column:actual_file_hash" json:"actual_file_hash,omitempty"` MimeType string `gorm:"column:mimetype" json:"mimetype,omitempty"` - AllocationRoot string `gorm:"column:allocation_root" json:"allocation_root,omitempty"` ThumbnailSize int64 `gorm:"column:thumbnail_size" json:"thumbnail_size,omitempty"` ThumbnailHash string `gorm:"column:thumbnail_hash" json:"thumbnail_hash,omitempty"` ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" json:"actual_thumbnail_size,omitempty"` ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"` EncryptedKey string `gorm:"column:encrypted_key" json:"encrypted_key,omitempty"` EncryptedKeyPoint string `gorm:"column:encrypted_key_point" json:"encrypted_key_point,omitempty"` + FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` CreatedAt common.Timestamp `gorm:"column:created_at" json:"created_at,omitempty"` UpdatedAt common.Timestamp `gorm:"column:updated_at" json:"updated_at,omitempty"` @@ -444,7 +439,6 @@ func (r *Ref) GetHashData() string { func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) { fr.FileMetaHash = encryption.Hash(fr.GetFileMetaHashData()) - fr.Hash = encryption.Hash(fr.GetFileHashData()) fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize))) fr.PathLevel = len(strings.Split(strings.TrimRight(fr.Path, "/"), "/")) fr.LookupHash = GetReferenceLookup(fr.AllocationID, fr.Path) @@ -453,12 +447,12 @@ func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool, collector Q if saveToDB && fr.HashToBeComputed { err = fr.SaveFileRef(ctx, collector) } - return fr.Hash, err + return fr.FileMetaHash, err } func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector QueryCollector) (h string, err error) { if !r.HashToBeComputed { - h = r.Hash + h = r.FileMetaHash return } @@ -471,7 +465,6 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que } }() - childHashes := make([]string, l) childFileMetaHashes := make([]string, l) var refNumBlocks, size, actualSize int64 @@ -484,20 +477,18 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que } childFileMetaHashes[i] = childRef.FileMetaHash - childHashes[i] = childRef.Hash refNumBlocks += childRef.NumBlocks size += childRef.Size actualSize += childRef.ActualFileSize } r.FileMetaHash = encryption.Hash(r.Path + strings.Join(childFileMetaHashes, ":")) - r.Hash = encryption.Hash(r.GetHashData() + strings.Join(childHashes, ":")) r.NumBlocks = refNumBlocks r.Size = size r.ActualFileSize = actualSize r.PathLevel = len(GetSubDirsFromPath(r.Path)) + 1 r.LookupHash = GetReferenceLookup(r.AllocationID, r.Path) - return r.Hash, err + return r.FileMetaHash, err } func (r *Ref) CalculateHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) { From 85ec94fe3450510f47ae857db407243d8530d7c1 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 11 Jul 2024 16:52:18 +0530 Subject: [PATCH 06/98] parent id --- .../blobbercore/allocation/copyfilechange.go | 7 --- .../allocation/file_changer_upload.go | 60 ++++++++++++------- .../blobbercore/allocation/movefilechange.go | 6 -- .../blobbercore/allocation/newdirchange.go | 7 --- .../blobbercore/reference/dbCollector.go | 6 -- .../0chain.net/blobbercore/reference/ref.go | 7 +-- 6 files changed, 42 insertions(+), 51 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 9f849df4e..23f69bcc1 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -77,12 +77,6 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, rootRef *reference.Re newRef := reference.NewDirectoryRef() newRef.AllocationID = rf.AllocationID newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/")) - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in file ID meta", newRef.Path)) - } - newRef.FileID = fileID newRef.ParentPath = filepath.Join("/", strings.Join(fields[:i], "/")) newRef.Name = fields[i] newRef.HashToBeComputed = true @@ -116,7 +110,6 @@ func (rf *CopyFileChange) processCopyRefs( return nil, common.NewError("invalid_parameter", fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) } - newRef.FileID = fileID newRef.ParentPath = destRef.Path newRef.CreatedAt = ts newRef.UpdatedAt = ts diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 290d4f4a1..a25d9c83c 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -3,7 +3,6 @@ package allocation import ( "context" "encoding/json" - "fmt" "path/filepath" "strings" @@ -62,31 +61,50 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, FilestoreVersion: filestore.VERSION, } - fileID, ok := fileIDMeta[newFile.Path] - if !ok || fileID == "" { - return common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newFile.Path)) - } - newFile.FileID = fileID - // find if ref exists - err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { - var refResult struct { - ID int64 - } - db := datastore.GetStore().GetTransaction(ctx) - err := db.Model(&reference.Ref{}).Select("id").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error - if err == gorm.ErrRecordNotFound { - collector.CreateRefRecord(newFile) - } else if err != nil { - return err - } + var refResult struct { + ID int64 + } + db := datastore.GetStore().GetTransaction(ctx) + err := db.Model(&reference.Ref{}).Select("id").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error + if err != nil && err != gorm.ErrRecordNotFound { + return err + } + if refResult.ID > 0 { deleteRecord := &reference.Ref{ ID: refResult.ID, } collector.DeleteRefRecord(deleteRecord) - return nil - }) + } + refResult.ID = 0 + // get parent id + parent := filepath.Dir(nf.Path) + parentLookupHash := reference.GetReferenceLookup(nf.AllocationID, parent) + err = db.Model(&reference.Ref{}).Select("id").Where("lookup_hash = ?", parentLookupHash).Take(&refResult).Error + if err != nil && err != gorm.ErrRecordNotFound { + return err + } + // create parent directory + if refResult.ID == 0 { + // get all parent directories + fields, err := common.GetParentPaths(filepath.Dir(parent)) + if err != nil { + return err + } + parentLookupHashes := make([]string, 0, len(fields)) + for i := 0; i < len(fields); i++ { + parentLookupHashes = append(parentLookupHashes, reference.GetReferenceLookup(nf.AllocationID, fields[i])) + + } + var parentRefs []reference.Ref + err = db.Model(&reference.Ref{}).Select("id", "path").Where("lookup_hash IN ?", parentLookupHashes).Order("path").Find(&parentRefs).Error + if err != nil && err != gorm.ErrRecordNotFound { + return err + } + } + + newFile.ParentID = refResult.ID + collector.CreateRefRecord(newFile) return err } diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange.go b/code/go/0chain.net/blobbercore/allocation/movefilechange.go index 7d3aa9c3e..5bdfee033 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange.go @@ -106,12 +106,6 @@ func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Re newRef.HashToBeComputed = true newRef.CreatedAt = ts newRef.UpdatedAt = ts - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.FileID = fileID dirRef.AddChild(newRef) dirRef = newRef } diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index d38e8135b..ea6af9130 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -3,7 +3,6 @@ package allocation import ( "context" "encoding/json" - "fmt" "path/filepath" "strings" "sync" @@ -34,12 +33,6 @@ func (nf *NewDir) ApplyChange(ctx context.Context, newRef.CreatedAt = ts newRef.UpdatedAt = ts newRef.HashToBeComputed = true - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.FileID = fileID err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { //check if ref exists exists, err := reference.IsRefExist(ctx, nf.AllocationID, newRef.Path) diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index 9c3c9b61e..37c3cc808 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -2,7 +2,6 @@ package reference import ( "context" - "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" ) @@ -16,7 +15,6 @@ type QueryCollector interface { type dbCollector struct { createdRefs []*Ref deletedRefs []*Ref - mut sync.Mutex } func NewCollector(changes int) QueryCollector { @@ -27,15 +25,11 @@ func NewCollector(changes int) QueryCollector { } func (dc *dbCollector) CreateRefRecord(ref *Ref) { - dc.mut.Lock() dc.createdRefs = append(dc.createdRefs, ref) - dc.mut.Unlock() } func (dc *dbCollector) DeleteRefRecord(ref *Ref) { - dc.mut.Lock() dc.deletedRefs = append(dc.deletedRefs, ref) - dc.mut.Unlock() } func (dc *dbCollector) Finalize(ctx context.Context) error { diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 516175618..8e80d104d 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -48,7 +48,7 @@ func init() { type Ref struct { ID int64 `gorm:"column:id;primaryKey"` - FileID string `gorm:"column:file_id" dirlist:"file_id" filelist:"file_id"` + ParentID int64 `gorm:"column:parent_id"` Type string `gorm:"column:type;size:1" dirlist:"type" filelist:"type"` AllocationID string `gorm:"column:allocation_id;size:64;not null;index:idx_path_alloc,priority:1;index:idx_parent_path_alloc,priority:1;index:idx_validation_alloc,priority:1" dirlist:"allocation_id" filelist:"allocation_id"` LookupHash string `gorm:"column:lookup_hash;size:64;not null;index:idx_lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash"` @@ -420,7 +420,7 @@ func (r *Ref) GetFileMetaHashData() string { func (fr *Ref) GetFileHashData() string { return fmt.Sprintf( - "%s:%s:%s:%s:%d:%d:%s:%d:%s", + "%s:%s:%s:%s:%d:%d:%s:%d", fr.AllocationID, fr.Type, // don't need to add it as well fr.Name, // don't see any utility as fr.Path below has name in it @@ -429,12 +429,11 @@ func (fr *Ref) GetFileHashData() string { fr.ActualFileSize, fr.ActualFileHash, fr.ChunkSize, - fr.FileID, ) } func (r *Ref) GetHashData() string { - return fmt.Sprintf("%s:%s:%s", r.AllocationID, r.Path, r.FileID) + return fmt.Sprintf("%s:%s", r.AllocationID, r.Path) } func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) { From a04e75861e11b6946c25369431e5770494549528 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 12 Jul 2024 16:38:18 +0530 Subject: [PATCH 07/98] mkdir and version marker --- .../allocation/allocationchange.go | 2 +- .../allocation/file_changer_upload.go | 25 ++--- .../handler/object_operation_handler.go | 20 ++++ .../0chain.net/blobbercore/reference/ref.go | 100 ++++++++++++++---- .../blobbercore/writemarker/version_marker.go | 58 ++++++++++ .../1698861371_full_db_snapshot.sql | 20 ++-- 6 files changed, 179 insertions(+), 46 deletions(-) create mode 100644 code/go/0chain.net/blobbercore/writemarker/version_marker.go diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 94d871a37..04b007c46 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -313,7 +313,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { tx := datastore.GetStore().GetTransaction(ctx) - err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "filestore_version").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). + err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "filestore_version", "lookup_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error { for _, ref := range refs { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index a25d9c83c..3cda08990 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -14,6 +14,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) // UploadFileChanger file change processor for continuous upload in INIT/APPEND/FINALIZE @@ -60,13 +61,15 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, PathLevel: len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")), FilestoreVersion: filestore.VERSION, } + newFile.FileMetaHash = encryption.Hash(newFile.GetFileMetaHashData()) // find if ref exists var refResult struct { - ID int64 + ID int64 + Type string } db := datastore.GetStore().GetTransaction(ctx) - err := db.Model(&reference.Ref{}).Select("id").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error + err := db.Model(&reference.Ref{}).Select("id", "type").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error if err != nil && err != gorm.ErrRecordNotFound { return err } @@ -80,26 +83,20 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, // get parent id parent := filepath.Dir(nf.Path) parentLookupHash := reference.GetReferenceLookup(nf.AllocationID, parent) - err = db.Model(&reference.Ref{}).Select("id").Where("lookup_hash = ?", parentLookupHash).Take(&refResult).Error + err = db.Model(&reference.Ref{}).Select("id", "type").Where("lookup_hash = ?", parentLookupHash).Take(&refResult).Error if err != nil && err != gorm.ErrRecordNotFound { return err } // create parent directory if refResult.ID == 0 { - // get all parent directories - fields, err := common.GetParentPaths(filepath.Dir(parent)) + parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent) if err != nil { return err } - parentLookupHashes := make([]string, 0, len(fields)) - for i := 0; i < len(fields); i++ { - parentLookupHashes = append(parentLookupHashes, reference.GetReferenceLookup(nf.AllocationID, fields[i])) - - } - var parentRefs []reference.Ref - err = db.Model(&reference.Ref{}).Select("id", "path").Where("lookup_hash IN ?", parentLookupHashes).Order("path").Find(&parentRefs).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err + refResult.ID = parentRef.ID + } else { + if refResult.Type != reference.DIRECTORY { + return common.NewError("invalid_parameter", "parent path is not a directory") } } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index ca4b2917c..61c708870 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -568,6 +568,21 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b fmt.Sprintf("Error while unmarshalling file ID meta data: %s", err.Error())) } + versionMarkerStr := r.FormValue("version_marker") + if versionMarkerStr == "" { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed") + } + versionMarker := writemarker.VersionMarker{} + err = json.Unmarshal([]byte(versionMarkerStr), &versionMarker) + if err != nil { + return nil, common.NewError("unmarshall_error", fmt.Sprintf("Error while unmarshalling version marker: %s", err.Error())) + } + + err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey) + if err != nil { + return nil, err + } + // Move preCommitDir to finalDir err = connectionObj.MoveToFilestore(ctx) if err != nil { @@ -587,6 +602,11 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b elapsedGetConnObj db := datastore.GetStore().GetTransaction(ctx) + err = db.Create(&versionMarker).Error + if err != nil { + return nil, common.NewError("db_error", fmt.Sprintf("Error while saving version marker: %s", err.Error())) + } + allocationObj.BlobberSizeUsed += connectionObj.Size allocationObj.UsedSize += connectionObj.Size diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 8e80d104d..e3995c62a 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -2,7 +2,6 @@ package reference import ( "context" - "errors" "fmt" "math" "path/filepath" @@ -155,41 +154,98 @@ func NewFileRef() *Ref { } // Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child +// func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { +// var dirRef *Ref +// db := datastore.GetStore().GetTransaction(ctx) +// // cleaning path to avoid edge case issues: append '/' prefix if not added and removing suffix '/' if added +// destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") +// dirs := strings.Split(destpath, "/") + +// for i := range dirs { +// currentPath := filepath.Join("/", filepath.Join(dirs[:i+1]...)) +// ref, err := GetReference(ctx, allocationID, currentPath) +// if err == nil { +// dirRef = ref +// continue +// } + +// if !errors.Is(err, gorm.ErrRecordNotFound) { +// // unexpected sql error +// return nil, err +// } + +// // dir doesn't exists , create it +// newRef := NewDirectoryRef() +// newRef.AllocationID = allocationID +// newRef.Path = currentPath +// newRef.ParentPath = filepath.Join("/", filepath.Join(dirs[:i]...)) +// newRef.Name = dirs[i] +// newRef.Type = DIRECTORY +// newRef.PathLevel = i + 1 +// newRef.LookupHash = GetReferenceLookup(allocationID, newRef.Path) +// err = db.Create(newRef).Error +// if err != nil { +// return nil, err +// } + +// dirRef = newRef +// } + +// return dirRef, nil +// } + func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { - var dirRef *Ref db := datastore.GetStore().GetTransaction(ctx) - // cleaning path to avoid edge case issues: append '/' prefix if not added and removing suffix '/' if added destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") - dirs := strings.Split(destpath, "/") + fields, err := common.GetParentPaths(filepath.Dir(destpath)) + if err != nil { + return nil, err + } - for i := range dirs { - currentPath := filepath.Join("/", filepath.Join(dirs[:i+1]...)) - ref, err := GetReference(ctx, allocationID, currentPath) - if err == nil { - dirRef = ref - continue - } + parentLookupHashes := make([]string, 0, len(fields)) + for i := 0; i < len(fields); i++ { + parentLookupHashes = append(parentLookupHashes, GetReferenceLookup(allocationID, fields[i])) + } - if !errors.Is(err, gorm.ErrRecordNotFound) { - // unexpected sql error - return nil, err + var parentRefs []*Ref + err = db.Model(&Ref{}).Select("id", "path", "type").Where("lookup_hash IN ?", parentLookupHashes).Order("path").Find(&parentRefs).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + var ( + parentID int64 + parentPath = "/" + ) + if len(parentRefs) > 0 { + parentID = parentRefs[len(parentRefs)-1].ID + parentPath = parentRefs[len(parentRefs)-1].Path + for i := 0; i < len(parentRefs); i++ { + if parentRefs[i].Type != DIRECTORY { + return nil, common.NewError("invalid_dir_tree", "DB has invalid tree.") + } } - - // dir doesn't exists , create it + } + for i := len(parentRefs); i < len(fields); i++ { newRef := NewDirectoryRef() newRef.AllocationID = allocationID - newRef.Path = currentPath - newRef.ParentPath = filepath.Join("/", filepath.Join(dirs[:i]...)) - newRef.Name = dirs[i] - newRef.Type = DIRECTORY + newRef.Path = fields[i] + newRef.ParentPath = parentPath + newRef.Name = filepath.Base(fields[i]) newRef.PathLevel = i + 1 - newRef.LookupHash = GetReferenceLookup(allocationID, newRef.Path) + newRef.ParentID = parentID + newRef.LookupHash = parentLookupHashes[i] err = db.Create(newRef).Error if err != nil { return nil, err } + parentID = newRef.ID + parentPath = newRef.Path + } - dirRef = newRef + dirRef := &Ref{ + AllocationID: allocationID, + ID: parentID, + Path: parentPath, } return dirRef, nil diff --git a/code/go/0chain.net/blobbercore/writemarker/version_marker.go b/code/go/0chain.net/blobbercore/writemarker/version_marker.go new file mode 100644 index 000000000..646725f19 --- /dev/null +++ b/code/go/0chain.net/blobbercore/writemarker/version_marker.go @@ -0,0 +1,58 @@ +package writemarker + +import ( + "context" + "fmt" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" +) + +type VersionMarker struct { + ID int64 `gorm:"column:sequence;primaryKey"` + AllocationID string `gorm:"allocation_id" json:"allocation_id"` + Version int64 `gorm:"version" json:"version"` + PreviousVersion int64 `gorm:"previous_version" json:"previous_version"` + Timestamp int64 `gorm:"timestamp" json:"timestamp"` + Signature string `gorm:"signature" json:"signature"` +} + +func (VersionMarker) TableName() string { + return "version_markers" +} + +func GetCurrentVersion(ctx context.Context, allocationID string) (*VersionMarker, error) { + db := datastore.GetStore().GetTransaction(ctx) + var vm VersionMarker + err := db.Where("allocation_id = ?", allocationID).Order("id DESC").Take(&vm).Error + return &vm, err +} + +func (vm *VersionMarker) Verify(allocationID, clientPubKey string) error { + if vm.AllocationID != allocationID { + return common.NewError("version_marker_validation_failed", "Invalid allocation id") + } + + if vm.Signature == "" { + return common.NewError("version_marker_validation_failed", "Signature is missing") + } + + hashData := vm.GetHashData() + signatureHash := encryption.Hash(hashData) + sigOK, err := encryption.Verify(clientPubKey, vm.Signature, signatureHash) + if err != nil { + return common.NewError("version_marker_validation_failed", "Error during verifying signature. "+err.Error()) + } + if !sigOK { + logging.Logger.Error("write_marker_sig_error", zap.Any("vm", vm)) + return common.NewError("version_marker_validation_failed", "Version marker signature is not valid") + } + return nil +} + +func (vm *VersionMarker) GetHashData() string { + return fmt.Sprintf("%s:%d:%d:%d", vm.AllocationID, vm.Version, vm.PreviousVersion, vm.Timestamp) +} diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 950dd0e6a..d45c4cb48 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -329,7 +329,6 @@ ALTER TABLE read_pools OWNER TO blobber_user; CREATE TABLE reference_objects ( id bigint NOT NULL, - file_id text, type character varying(1), allocation_id character varying(64) NOT NULL, lookup_hash character varying(64) NOT NULL, @@ -337,25 +336,17 @@ CREATE TABLE reference_objects ( thumbnail_filename text, path character varying(1000) NOT NULL COLLATE pg_catalog."POSIX", file_meta_hash character varying(64) NOT NULL, - hash character varying(64) NOT NULL, num_of_blocks bigint DEFAULT 0 NOT NULL, - path_hash character varying(64) NOT NULL, parent_path character varying(999), level bigint DEFAULT 0 NOT NULL, custom_meta text NOT NULL, - validation_root character varying(64) NOT NULL, - prev_validation_root text, - validation_root_signature character varying(64), size bigint DEFAULT 0 NOT NULL, - fixed_merkle_root character varying(64) NOT NULL, actual_file_size bigint DEFAULT 0 NOT NULL, actual_file_hash_signature character varying(64), actual_file_hash character varying(64) NOT NULL, mimetype character varying(255) NOT NULL, - allocation_root character varying(64) NOT NULL, thumbnail_size bigint DEFAULT 0 NOT NULL, thumbnail_hash character varying(64) NOT NULL, - prev_thumbnail_hash text, actual_thumbnail_size bigint DEFAULT 0 NOT NULL, actual_thumbnail_hash character varying(64) NOT NULL, encrypted_key character varying(64), @@ -367,6 +358,9 @@ CREATE TABLE reference_objects ( chunk_size bigint DEFAULT 65536 NOT NULL, num_of_updates bigint, num_of_block_downloads bigint + data_hash character varying(64), + data_hash_signature character varying(64), + parent_id bigint NOT NULL ); @@ -854,6 +848,14 @@ ALTER TABLE ONLY allocation_changes ADD CONSTRAINT fk_allocation_connections_changes FOREIGN KEY (connection_id) REFERENCES allocation_connections(id) ON DELETE CASCADE; + -- + -- Name: fk_reference_objects; TYPE FK CONSTRAINT; Schema: public; Owner: blobber_user + -- + + ALTER TABLE ONLY reference_objects + ADD CONSTRAINT fk_reference_objects FOREIGN KEY (parent_id) REFERENCES reference_objects(id) ON DELETE CASCADE; + + -- -- Name: connection_id_index; Type: INDEX; Schema: public; Owner: blobber_user -- From e012e663e4495924c5e96851d0994148c75ad763 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 15 Jul 2024 13:29:08 +0530 Subject: [PATCH 08/98] add version markers table --- .../1698861371_full_db_snapshot.sql | 2 +- .../migrations/1721021811_version_marker.sql | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 goose/migrations/1721021811_version_marker.sql diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index d45c4cb48..b2f7fb874 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -375,7 +375,7 @@ CREATE SEQUENCE reference_objects_id_seq INCREMENT BY 1 NO MINVALUE NO MAXVALUE - CACHE 1; + CACHE 100; ALTER TABLE reference_objects_id_seq OWNER TO blobber_user; diff --git a/goose/migrations/1721021811_version_marker.sql b/goose/migrations/1721021811_version_marker.sql new file mode 100644 index 000000000..d439ad90f --- /dev/null +++ b/goose/migrations/1721021811_version_marker.sql @@ -0,0 +1,44 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE TABLE version_markers( + id bigint NOT NULL, + allocation_id character varying(64) NOT NULL, + version bigint NOT NULL, + previous_version bigint NOT NULL, + "timestamp" bigint NOT NULL, + signature character varying(64), +); + +ALTER TABLE version_markers OWNER TO blobber_user; + +-- +-- Name: version_markers_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user +-- + +CREATE SEQUENCE version_markers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER TABLE version_markers_id_seq OWNER TO blobber_user; + + +-- +-- Name: version_markers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user +-- + +ALTER SEQUENCE version_markers_id_seq OWNED BY version_markers.id; + + +-- +-- Name: version_markers version_markers_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user +-- + +ALTER TABLE ONLY version_markers + ADD CONSTRAINT version_markers_pkey PRIMARY KEY (id); + + +-- +goose StatementEnd From 23e7eb9d2fcc90cb9902ec9e4e84c1f91d6093dc Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Mon, 22 Jul 2024 02:14:38 +0530 Subject: [PATCH 09/98] add repair in version marker --- .../0chain.net/blobbercore/reference/object.go | 3 +++ .../blobbercore/writemarker/version_marker.go | 16 +++++++++------- goose/migrations/1698861371_full_db_snapshot.sql | 6 ++++++ goose/migrations/1718391849_ref_index.sql | 2 ++ goose/migrations/1721021811_version_marker.sql | 8 +++++--- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index aad965182..317dea4b0 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -7,6 +7,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) func DeleteObject(ctx context.Context, allocationID, objPath string, ts common.Timestamp) error { @@ -22,6 +24,7 @@ func DeleteObject(ctx context.Context, allocationID, objPath string, ts common.T Valid: true, }, allocationID, "/", objPath, likePath).Error if err != nil { + logging.Logger.Error("delete_object_error", zap.Error(err)) return err } return err diff --git a/code/go/0chain.net/blobbercore/writemarker/version_marker.go b/code/go/0chain.net/blobbercore/writemarker/version_marker.go index 646725f19..805a8a4f4 100644 --- a/code/go/0chain.net/blobbercore/writemarker/version_marker.go +++ b/code/go/0chain.net/blobbercore/writemarker/version_marker.go @@ -12,12 +12,14 @@ import ( ) type VersionMarker struct { - ID int64 `gorm:"column:sequence;primaryKey"` - AllocationID string `gorm:"allocation_id" json:"allocation_id"` - Version int64 `gorm:"version" json:"version"` - PreviousVersion int64 `gorm:"previous_version" json:"previous_version"` - Timestamp int64 `gorm:"timestamp" json:"timestamp"` - Signature string `gorm:"signature" json:"signature"` + ID int64 `gorm:"column:sequence;primaryKey"` + AllocationID string `gorm:"allocation_id" json:"allocation_id"` + Version int64 `gorm:"version" json:"version"` + Timestamp int64 `gorm:"timestamp" json:"timestamp"` + Signature string `gorm:"signature" json:"signature"` + IsRepair bool `gorm:"is_repair" json:"is_repair"` + RepairVersion int64 `gorm:"repair_version" json:"repair_version"` + RepairOffset string `gorm:"repair_offset" json:"repair_offset"` } func (VersionMarker) TableName() string { @@ -54,5 +56,5 @@ func (vm *VersionMarker) Verify(allocationID, clientPubKey string) error { } func (vm *VersionMarker) GetHashData() string { - return fmt.Sprintf("%s:%d:%d:%d", vm.AllocationID, vm.Version, vm.PreviousVersion, vm.Timestamp) + return fmt.Sprintf("%s:%d:%d", vm.AllocationID, vm.Version, vm.Timestamp) } diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index b2f7fb874..7448ef5cc 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -783,6 +783,12 @@ CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english': CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path); +-- +-- Name: idx_parent_id_alloc; Type: INDEX; Schema: public; Owner: blobber_user +-- + +CREATE INDEX idx_parent_id_alloc ON reference_objects USING btree (allocation_id, parent_id) where deleted_at is NULL; + -- -- Name: idx_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index 4380bc232..09d6570e6 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -1,4 +1,6 @@ -- +goose Up -- +goose StatementBegin DROP INDEX idx_created_at,idx_updated_at,idx_lookup_hash,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; + +CREATE INDEX idx_allocation_precommit_deleted ON reference_objects (allocation_id, is_precommit) where deleted_at is NULL; -- +goose StatementEnd \ No newline at end of file diff --git a/goose/migrations/1721021811_version_marker.sql b/goose/migrations/1721021811_version_marker.sql index d439ad90f..4fd0f3e94 100644 --- a/goose/migrations/1721021811_version_marker.sql +++ b/goose/migrations/1721021811_version_marker.sql @@ -4,10 +4,12 @@ CREATE TABLE version_markers( id bigint NOT NULL, allocation_id character varying(64) NOT NULL, - version bigint NOT NULL, - previous_version bigint NOT NULL, + "version" bigint NOT NULL, "timestamp" bigint NOT NULL, signature character varying(64), + is_repair boolean NOT NULL DEFAULT false, + repair_version bigint, + repair_offset character varying(1000) ); ALTER TABLE version_markers OWNER TO blobber_user; @@ -21,7 +23,7 @@ CREATE SEQUENCE version_markers_id_seq INCREMENT BY 1 NO MINVALUE NO MAXVALUE - CACHE 1; + CACHE 100; ALTER TABLE version_markers_id_seq OWNER TO blobber_user; From 7e3540dfcf15a3f3ac09882aa2c872cea086b0c9 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 03:18:15 +0530 Subject: [PATCH 10/98] add prev size and add vm to rollback --- .../blobbercore/allocation/entity.go | 25 ++--- .../blobbercore/blobberhttp/response.go | 4 + .../handler/object_operation_handler.go | 94 +++++-------------- .../blobbercore/handler/storage_handler.go | 41 ++------ .../blobbercore/writemarker/version_marker.go | 11 ++- .../1698861371_full_db_snapshot.sql | 1 + .../migrations/1721021811_version_marker.sql | 4 + 7 files changed, 66 insertions(+), 114 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 50766c38c..b752cafff 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -43,23 +43,26 @@ type Allocation struct { Tx string `gorm:"column:tx;size:64;not null;unique;index:idx_unique_allocations_tx,unique"` TotalSize int64 `gorm:"column:size;not null;default:0"` UsedSize int64 `gorm:"column:used_size;not null;default:0"` + PrevUsedSize int64 `gorm:"column:prev_used_size;not null;default:0"` OwnerID string `gorm:"column:owner_id;size:64;not null"` OwnerPublicKey string `gorm:"column:owner_public_key;size:512;not null"` RepairerID string `gorm:"column:repairer_id;size:64;not null"` Expiration common.Timestamp `gorm:"column:expiration_date;not null"` // AllocationRoot allcation_root of last write_marker - AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` - FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` - BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` - BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` - LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` - LastRedeemedSeq int64 `gorm:"column:last_redeemed_sequence;default:0"` - IsRedeemRequired bool `gorm:"column:is_redeem_required"` - TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` - StartTime common.Timestamp `gorm:"column:start_time;not null"` + AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` + FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` + BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` + BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` + PrevBlobberSizeUsed int64 `gorm:"column:prev_blobber_size_used;not null;default:0"` + LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` + LastRedeemedSeq int64 `gorm:"column:last_redeemed_sequence;default:0"` + IsRedeemRequired bool `gorm:"column:is_redeem_required"` + TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` + StartTime common.Timestamp `gorm:"column:start_time;not null"` // Ending and cleaning - CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` - Finalized bool `gorm:"column:finalized;not null;default:false"` + CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` + Finalized bool `gorm:"column:finalized;not null;default:false"` + AllocationVersion int64 `gorm:"column:allocation_version;not null;default:0"` // FileOptions to define file restrictions on an allocation for third-parties // default 00000000 for all crud operations suggesting only owner has the below listed abilities. diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index 47dd59ad2..49359e806 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -68,3 +68,7 @@ type LatestWriteMarkerResult struct { PrevWM *writemarker.WriteMarker `json:"prev_write_marker"` Version string `json:"version"` } + +type LatestVersionMarkerResult struct { + VersionMarker *writemarker.VersionMarker `json:"version_marker"` +} diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 61c708870..9835fadd6 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -606,7 +606,8 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b if err != nil { return nil, common.NewError("db_error", fmt.Sprintf("Error while saving version marker: %s", err.Error())) } - + allocationObj.PrevBlobberSizeUsed = allocationObj.BlobberSizeUsed + allocationObj.PrevUsedSize = allocationObj.UsedSize allocationObj.BlobberSizeUsed += connectionObj.Size allocationObj.UsedSize += connectionObj.Size @@ -614,11 +615,17 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b "used_size": allocationObj.UsedSize, "blobber_size_used": allocationObj.BlobberSizeUsed, "is_redeem_required": true, + "allocation_version": versionMarker.Version, + "prev_used_size": allocationObj.PrevUsedSize, + "prev_blobber_size": allocationObj.PrevBlobberSizeUsed, } updateOption := func(a *allocation.Allocation) { a.IsRedeemRequired = true a.BlobberSizeUsed = allocationObj.BlobberSizeUsed a.UsedSize = allocationObj.UsedSize + a.AllocationVersion = versionMarker.Version + a.PrevUsedSize = allocationObj.PrevUsedSize + a.PrevBlobberSizeUsed = allocationObj.PrevBlobberSizeUsed } if err = allocation.Repo.UpdateAllocation(ctx, allocationObj, updateMap, updateOption); err != nil { @@ -1237,19 +1244,14 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } - if allocationObj.AllocationRoot == "" { - Logger.Error("Allocation root is not set", zap.String("allocation_id", allocationObj.ID)) - return nil, common.NewError("invalid_parameters", "Allocation root is not set") + if allocationObj.AllocationVersion == 0 { + Logger.Error("Allocation version is 0", zap.String("allocation_id", allocationObj.ID)) + return nil, common.NewError("invalid_parameters", "Allocation version is not set") } elapsedAllocation := time.Since(startTime) allocationID := allocationObj.ID - connectionID, ok := common.GetField(r, "connection_id") - if !ok { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - elapsedGetLock := time.Since(startTime) - elapsedAllocation if clientID == "" || clientKey == "" { @@ -1260,9 +1262,9 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - writeMarkerString := r.FormValue("write_marker") - writeMarker := writemarker.WriteMarker{} - err = json.Unmarshal([]byte(writeMarkerString), &writeMarker) + versionMarkerString := r.FormValue("version_marker") + versionMarker := writemarker.VersionMarker{} + err = json.Unmarshal([]byte(versionMarkerString), &versionMarker) if err != nil { return nil, common.NewErrorf("invalid_parameters", "Invalid parameters. Error parsing the writemarker for commit: %v", @@ -1270,40 +1272,9 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob } var result blobberhttp.CommitResult + err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey) - var latestWriteMarkerEntity *writemarker.WriteMarkerEntity - latestWriteMarkerEntity, err = writemarker.GetWriteMarkerEntity(ctx, - allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewErrorf("latest_write_marker_read_error", - "Error reading the latest write marker for allocation: %v", err) - } - if latestWriteMarkerEntity == nil { - return nil, common.NewError("latest_write_marker_not_found", - "Latest write marker not found for allocation") - } - - writemarkerEntity := &writemarker.WriteMarkerEntity{} - writemarkerEntity.WM = writeMarker - - err = writemarkerEntity.VerifyRollbackMarker(ctx, allocationObj, latestWriteMarkerEntity) - if err != nil { - return nil, common.NewError("write_marker_verification_failed", "Verification of the write marker failed: "+err.Error()) - } - - if writemarkerEntity.WM.ChainLength > config.Configuration.MaxChainLength { - return nil, common.NewError("chain_length_exceeded", "Chain length exceeded") - } - - elapsedVerifyWM := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - var clientIDForWriteRedeem = writeMarker.ClientID - - if err := writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil { - return nil, err - } - - elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM + elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock timeoutCtx, cancel := context.WithTimeout(ctx, 45*time.Second) defer cancel() c := datastore.GetStore().CreateTransaction(timeoutCtx) @@ -1313,7 +1284,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob txn.Rollback() return nil, common.NewError("allocation_rollback_error", "Error applying the rollback for allocation: "+err.Error()) } - elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - elapsedWritePreRedeem + elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem //get allocation root and ref rootRef, err := reference.GetLimitedRefFieldsByPath(c, allocationID, "/", []string{"hash", "file_meta_hash", "is_precommit"}) @@ -1326,22 +1297,6 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob } Logger.Info("rollback_root_ref", zap.Any("root_ref", rootRef)) - fileMetaRoot := rootRef.FileMetaHash - if fileMetaRoot != writeMarker.FileMetaRoot { - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity - } - result.Success = false - result.ErrorMessage = "File meta root in the write marker does not match the calculated file meta root." + - " Expected hash: " + fileMetaRoot + "; Got: " + writeMarker.FileMetaRoot - txn.Rollback() - return &result, common.NewError("file_meta_root_mismatch", result.ErrorMessage) - } - - writemarkerEntity.ConnectionID = connectionID - writemarkerEntity.ClientPublicKey = clientKey - Logger.Info("rollback_writemarker", zap.Any("writemarker", writemarkerEntity.WM)) - alloc, err := allocation.Repo.GetByIdAndLock(c, allocationID) Logger.Info("[rollback]Lock Allocation", zap.Bool("is_redeem_required", alloc.IsRedeemRequired), zap.String("allocation_root", alloc.AllocationRoot), zap.String("latest_wm_redeemed", alloc.LatestRedeemedWM)) if err != nil { @@ -1349,15 +1304,15 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return &result, common.NewError("allocation_read_error", "Error reading the allocation object") } - alloc.BlobberSizeUsed -= latestWriteMarkerEntity.WM.Size - alloc.UsedSize -= latestWriteMarkerEntity.WM.Size - alloc.FileMetaRoot = fileMetaRoot + alloc.BlobberSizeUsed = alloc.PrevBlobberSizeUsed + alloc.UsedSize = alloc.PrevUsedSize alloc.IsRedeemRequired = true + alloc.AllocationVersion = versionMarker.Version updateMap := map[string]interface{}{ "blobber_size_used": alloc.BlobberSizeUsed, "used_size": alloc.UsedSize, - "file_meta_root": alloc.FileMetaRoot, "is_redeem_required": true, + "allocation_version": versionMarker.Version, } updateOption := func(a *allocation.Allocation) { @@ -1367,8 +1322,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob a.FileMetaRoot = alloc.FileMetaRoot a.IsRedeemRequired = alloc.IsRedeemRequired } - writemarkerEntity.Latest = true - err = txn.Create(writemarkerEntity).Error + err = txn.Create(versionMarker).Error if err != nil { txn.Rollback() return &result, common.NewError("write_marker_error", "Error persisting the write marker "+err.Error()) @@ -1387,9 +1341,8 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob Logger.Error("Error committing the rollback for allocation", zap.Error(err)) } - elapsedCommitRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - elapsedWritePreRedeem + elapsedCommitRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem result.AllocationRoot = allocationObj.AllocationRoot - result.WriteMarker = writemarkerEntity result.Success = true result.ErrorMessage = "" commitOperation := "rollback" @@ -1398,7 +1351,6 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob zap.String("alloc_id", allocationID), zap.Duration("get_alloc", elapsedAllocation), zap.Duration("get-lock", elapsedGetLock), - zap.Duration("verify-wm", elapsedVerifyWM), zap.Duration("write-pre-redeem", elapsedWritePreRedeem), zap.Duration("apply-rollback", elapsedApplyRollback), zap.Duration("total", time.Since(startTime)), diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 2cf444e92..a1b98c455 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -418,7 +418,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* return &result, nil } -func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Request) (*blobberhttp.LatestWriteMarkerResult, error) { +func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Request) (*blobberhttp.LatestVersionMarkerResult, error) { clientID := ctx.Value(constants.ContextKeyClient).(string) if clientID == "" { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") @@ -439,41 +439,20 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req return nil, common.NewError("invalid_signature", "could not verify the allocation owner") } - var latestWM *writemarker.WriteMarkerEntity - var prevWM *writemarker.WriteMarkerEntity - if allocationObj.AllocationRoot == "" { - latestWM = nil - } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) + var vm *writemarker.VersionMarker + if allocationObj.AllocationVersion != 0 { + vm, err = writemarker.GetVersionMarker(ctx, allocationId, allocationObj.AllocationVersion) if err != nil { - Logger.Error("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID)) - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation. "+err.Error()) - } - if latestWM == nil { - Logger.Info("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID)) - return nil, common.NewError("latest_write_marker_read_error", "Latest write marker not found for allocation.") - } - if latestWM.WM.PreviousAllocationRoot != "" { - prevWM, err = writemarker.GetWriteMarkerEntity(ctx, latestWM.WM.PreviousAllocationRoot) - if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the previous write marker for allocation."+err.Error()) - } + return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) } + } else { + vm = &writemarker.VersionMarker{} } - var result blobberhttp.LatestWriteMarkerResult - result.Version = writemarker.MARKER_VERSION - if latestWM != nil { - if latestWM.Status == writemarker.Committed { - latestWM.WM.ChainLength = 0 // start a new chain - } - result.LatestWM = &latestWM.WM + result := &blobberhttp.LatestVersionMarkerResult{ + VersionMarker: vm, } - if prevWM != nil { - result.PrevWM = &prevWM.WM - } - - return &result, nil + return result, nil } func (fsh *StorageHandler) GetReferencePath(ctx context.Context, r *http.Request) (*blobberhttp.ReferencePathResult, error) { diff --git a/code/go/0chain.net/blobbercore/writemarker/version_marker.go b/code/go/0chain.net/blobbercore/writemarker/version_marker.go index 805a8a4f4..b1dbdcfaf 100644 --- a/code/go/0chain.net/blobbercore/writemarker/version_marker.go +++ b/code/go/0chain.net/blobbercore/writemarker/version_marker.go @@ -13,6 +13,8 @@ import ( type VersionMarker struct { ID int64 `gorm:"column:sequence;primaryKey"` + ClientID string `gorm:"client_id" json:"client_id"` + BlobberID string `gorm:"blobber_id" json:"blobber_id"` AllocationID string `gorm:"allocation_id" json:"allocation_id"` Version int64 `gorm:"version" json:"version"` Timestamp int64 `gorm:"timestamp" json:"timestamp"` @@ -33,6 +35,13 @@ func GetCurrentVersion(ctx context.Context, allocationID string) (*VersionMarker return &vm, err } +func GetVersionMarker(ctx context.Context, allocationID string, version int64) (*VersionMarker, error) { + db := datastore.GetStore().GetTransaction(ctx) + var vm VersionMarker + err := db.Where("allocation_id = ? and version = ?", allocationID, version).Order("id DESC").Take(&vm).Error + return &vm, err +} + func (vm *VersionMarker) Verify(allocationID, clientPubKey string) error { if vm.AllocationID != allocationID { return common.NewError("version_marker_validation_failed", "Invalid allocation id") @@ -56,5 +65,5 @@ func (vm *VersionMarker) Verify(allocationID, clientPubKey string) error { } func (vm *VersionMarker) GetHashData() string { - return fmt.Sprintf("%s:%d:%d", vm.AllocationID, vm.Version, vm.Timestamp) + return fmt.Sprintf("%s:%s:%s:%d:%d", vm.AllocationID, vm.ClientID, vm.BlobberID, vm.Version, vm.Timestamp) } diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 7448ef5cc..0e384912d 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -104,6 +104,7 @@ CREATE TABLE allocations ( finalized boolean DEFAULT false NOT NULL, file_options integer DEFAULT 63 NOT NULL, start_time bigint NOT NULL + allocation_version bigint DEFAULT 0 NOT NULL ); diff --git a/goose/migrations/1721021811_version_marker.sql b/goose/migrations/1721021811_version_marker.sql index 4fd0f3e94..4b035cef5 100644 --- a/goose/migrations/1721021811_version_marker.sql +++ b/goose/migrations/1721021811_version_marker.sql @@ -4,6 +4,8 @@ CREATE TABLE version_markers( id bigint NOT NULL, allocation_id character varying(64) NOT NULL, + blobber_id character varying(64) NOT NULL, + client_id character varying(64) NOT NULL, "version" bigint NOT NULL, "timestamp" bigint NOT NULL, signature character varying(64), @@ -43,4 +45,6 @@ ALTER TABLE ONLY version_markers ADD CONSTRAINT version_markers_pkey PRIMARY KEY (id); +CREATE INDEX version_markers_allocation_id_idx ON version_markers USING btree (allocation_id,version); + -- +goose StatementEnd From 7454b0b22e3d9f0b15c5309e0f08c5da63b2dab5 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 13:43:16 +0530 Subject: [PATCH 11/98] fix sql migration and use pointer for parent id --- .../blobbercore/allocation/file_changer_upload.go | 2 +- code/go/0chain.net/blobbercore/reference/ref.go | 8 ++++++-- goose/migrations/1698861371_full_db_snapshot.sql | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 3cda08990..b6558b06c 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -100,7 +100,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, } } - newFile.ParentID = refResult.ID + newFile.ParentID = &refResult.ID collector.CreateRefRecord(newFile) return err diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index e3995c62a..6d412c375 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -47,7 +47,7 @@ func init() { type Ref struct { ID int64 `gorm:"column:id;primaryKey"` - ParentID int64 `gorm:"column:parent_id"` + ParentID *int64 `gorm:"column:parent_id"` Type string `gorm:"column:type;size:1" dirlist:"type" filelist:"type"` AllocationID string `gorm:"column:allocation_id;size:64;not null;index:idx_path_alloc,priority:1;index:idx_parent_path_alloc,priority:1;index:idx_validation_alloc,priority:1" dirlist:"allocation_id" filelist:"allocation_id"` LookupHash string `gorm:"column:lookup_hash;size:64;not null;index:idx_lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash"` @@ -226,13 +226,17 @@ func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { } } for i := len(parentRefs); i < len(fields); i++ { + var parentIDRef *int64 + if parentID > 0 { + parentIDRef = &parentID + } newRef := NewDirectoryRef() newRef.AllocationID = allocationID newRef.Path = fields[i] newRef.ParentPath = parentPath newRef.Name = filepath.Base(fields[i]) newRef.PathLevel = i + 1 - newRef.ParentID = parentID + newRef.ParentID = parentIDRef newRef.LookupHash = parentLookupHashes[i] err = db.Create(newRef).Error if err != nil { diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 0e384912d..ea3e02fdf 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -103,7 +103,7 @@ CREATE TABLE allocations ( cleaned_up boolean DEFAULT false NOT NULL, finalized boolean DEFAULT false NOT NULL, file_options integer DEFAULT 63 NOT NULL, - start_time bigint NOT NULL + start_time bigint NOT NULL, allocation_version bigint DEFAULT 0 NOT NULL ); @@ -361,7 +361,7 @@ CREATE TABLE reference_objects ( num_of_block_downloads bigint data_hash character varying(64), data_hash_signature character varying(64), - parent_id bigint NOT NULL + parent_id bigint DEFAULT NULL ); From c0cfba3bbf0cb48570adf967c3b14da5f1571a11 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 14:35:12 +0530 Subject: [PATCH 12/98] fix ref table --- goose/migrations/1698861371_full_db_snapshot.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index ea3e02fdf..1ad55ef36 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -358,7 +358,7 @@ CREATE TABLE reference_objects ( is_precommit boolean DEFAULT false NOT NULL, chunk_size bigint DEFAULT 65536 NOT NULL, num_of_updates bigint, - num_of_block_downloads bigint + num_of_block_downloads bigint, data_hash character varying(64), data_hash_signature character varying(64), parent_id bigint DEFAULT NULL From 854fac7d663eb8d6856d8e2b0ea3606c0c953006 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 15:48:46 +0530 Subject: [PATCH 13/98] fix allocation table --- goose/migrations/1698861371_full_db_snapshot.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 1ad55ef36..086f3918d 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -104,7 +104,9 @@ CREATE TABLE allocations ( finalized boolean DEFAULT false NOT NULL, file_options integer DEFAULT 63 NOT NULL, start_time bigint NOT NULL, - allocation_version bigint DEFAULT 0 NOT NULL + allocation_version bigint DEFAULT 0 NOT NULL, + prev_used_size bigint DEFAULT 0 NOT NULL, + prev_blobber_size_used bigint DEFAULT 0 NOT NULL ); From 8ff1e386aa28614f35bad940d7688aa11a4137ac Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 16:30:41 +0530 Subject: [PATCH 14/98] add info log --- code/go/0chain.net/blobbercore/allocation/allocationchange.go | 1 + code/go/0chain.net/blobbercore/handler/file_command_upload.go | 1 + .../0chain.net/blobbercore/handler/object_operation_handler.go | 1 + 3 files changed, 3 insertions(+) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 04b007c46..ecbc299a6 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -154,6 +154,7 @@ func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clien ).Preload("Changes").Take(cc).Error if err == nil { + logging.Logger.Info("getAllocationChanges", zap.String("connection_id", connectionID), zap.Int("changes", len(cc.Changes))) cc.ComputeProperties() // Load connection Obj size from memory cc.Size = GetConnectionObjSize(connectionID) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 0ae212471..aee74c898 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -219,6 +219,7 @@ func (cmd *UploadFileCommand) AddChange(ctx context.Context) error { connectionInput, _ := cmd.fileChanger.Marshal() cmd.allocationChange.LookupHash = reference.GetReferenceLookup(cmd.fileChanger.AllocationID, cmd.fileChanger.Path) cmd.allocationChange.Input = connectionInput + logging.Logger.Info("AddChange: ", zap.String("connectionID", cmd.allocationChange.ConnectionID)) return cmd.allocationChange.Create(ctx) } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 9835fadd6..c6ce87748 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -536,6 +536,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b "Invalid connection id. Connection id was not found: %v", err) } if len(connectionObj.Changes) == 0 { + logging.Logger.Info("commit_write", zap.String("connection_id", connectionID)) if connectionObj.Status == allocation.NewConnection { return nil, common.NewError("invalid_parameters", "Invalid connection id. Connection not found.") From de07b61f3fbc9fd4a1aba76fea0616a4e696b2a7 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 17:18:46 +0530 Subject: [PATCH 15/98] fix upload validation --- code/go/0chain.net/blobbercore/handler/file_command_upload.go | 2 +- code/go/0chain.net/blobbercore/handler/handler_middlewares.go | 2 +- .../0chain.net/blobbercore/handler/object_operation_handler.go | 2 +- code/go/0chain.net/blobbercore/handler/tests_common_test.go | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index aee74c898..3eb111fef 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -80,7 +80,7 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_connection", "Invalid connection id") } - fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, cmd.fileChanger.Path) + fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, fileChanger.Path) thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) if thumbHeader != nil { diff --git a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go index 424b4c99f..bd073d88e 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go +++ b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go @@ -41,7 +41,7 @@ func UseRecovery(h http.Handler) http.Handler { defer func() { if err := recover(); err != nil { escapedUrl := sanitizeString(r.URL.String()) - logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err)) + logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err), zap.Stack("recover_stack")) } }() diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 77475f2fe..854a97610 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1231,7 +1231,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*all if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } - + logging.Logger.Info("writeFileAllocation") if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") } diff --git a/code/go/0chain.net/blobbercore/handler/tests_common_test.go b/code/go/0chain.net/blobbercore/handler/tests_common_test.go index 78fa2dcb7..791268833 100644 --- a/code/go/0chain.net/blobbercore/handler/tests_common_test.go +++ b/code/go/0chain.net/blobbercore/handler/tests_common_test.go @@ -79,7 +79,6 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string, Name: fileData.Name, Path: fileData.Path, FixedMerkleRoot: "", - ValidationRoot: fileData.ValidationRoot, Size: n, }, nil } From 75c745c20a463f6fbdda26195a40b81f238d311a Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 17:38:07 +0530 Subject: [PATCH 16/98] remove fileMeta --- .../allocation/file_changer_upload.go | 2 +- .../blobbercore/allocation/newdirchange.go | 29 ++----------------- .../handler/object_operation_handler.go | 11 ------- 3 files changed, 3 insertions(+), 39 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index b721fc54d..3c68a477b 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -25,7 +25,7 @@ type UploadFileChanger struct { // ApplyChange update references, and create a new FileRef func (nf *UploadFileChanger) applyChange(ctx context.Context, - ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { + ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { if nf.AllocationID == "" { return common.NewError("invalid_parameter", "allocation_id is required") diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index ea6af9130..3e6505bad 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -3,11 +3,8 @@ package allocation import ( "context" "encoding/json" - "path/filepath" - "strings" "sync" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -20,30 +17,8 @@ type NewDir struct { } func (nf *NewDir) ApplyChange(ctx context.Context, - ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { - - newRef := reference.NewDirectoryRef() - newRef.AllocationID = nf.AllocationID - newRef.Path = nf.Path - newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, newRef.Path) - newRef.PathLevel = len(strings.Split(strings.TrimRight(newRef.Path, "/"), "/")) - newRef.ParentPath = filepath.Dir(newRef.Path) - newRef.Name = filepath.Base(newRef.Path) - newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, newRef.Path) - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - newRef.HashToBeComputed = true - err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { - //check if ref exists - exists, err := reference.IsRefExist(ctx, nf.AllocationID, newRef.Path) - if err != nil { - return err - } - if !exists { - collector.CreateRefRecord(newRef) - } - return nil - }) + ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + _, err := reference.Mkdir(ctx, nf.AllocationID, nf.Path) return err } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 854a97610..de8595a4a 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -646,18 +646,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } var result blobberhttp.CommitResult - - fileIDMetaStr := r.FormValue("file_id_meta") - if fileIDMetaStr == "" { - return nil, common.NewError("invalid_parameters", "Invalid file ID meta passed") - } fileIDMeta := make(map[string]string, 0) - err = json.Unmarshal([]byte(fileIDMetaStr), &fileIDMeta) - if err != nil { - return nil, common.NewError("unmarshall_error", - fmt.Sprintf("Error while unmarshalling file ID meta data: %s", err.Error())) - } - versionMarkerStr := r.FormValue("version_marker") if versionMarkerStr == "" { return nil, common.NewError("invalid_parameters", "Invalid version marker passed") From f8de5a913600d13059937464a02e01c40bf1b079 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 19:45:49 +0530 Subject: [PATCH 17/98] init hasher --- .../blobbercore/allocation/connection.go | 4 ++++ .../allocation/file_changer_upload_test.go | 2 +- .../allocation/updatefilechange_test.go | 2 +- .../0chain.net/blobbercore/filestore/store_test.go | 14 +++++++------- .../blobbercore/filestore/tree_validation.go | 2 +- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index 5b731e52b..75ac30b0c 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -190,6 +190,10 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string if err != nil { return saveChange, err } + hasher := filestore.NewCommitHasher(contentSize) + change.hasher = hasher + change.seqPQ = seqpriorityqueue.NewSeqPriorityQueue(contentSize) + go hasher.Start(connectionObj.ctx, connectionID, connectionObj.AllocationID, fileName, pathHash, change.seqPQ) saveChange = true } else { change.lock.Lock() diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go index 4ea4e3bce..dced40d75 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go @@ -106,7 +106,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { changes: make(map[string]*ConnectionChange), } connectionProcessor["connection_id"].changes[pathHash] = &ConnectionChange{ - hasher: filestore.GetNewCommitHasher(2310), + hasher: filestore.NewCommitHasher(2310), } change := &UploadFileChanger{ BaseFileChanger: BaseFileChanger{ diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index da41fcc62..5b4f4e162 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -344,7 +344,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { changes: make(map[string]*ConnectionChange), } connectionProcessor["connection_id"].changes[pathHash] = &ConnectionChange{ - hasher: filestore.GetNewCommitHasher(2310), + hasher: filestore.NewCommitHasher(2310), } change := &UpdateFileChanger{ BaseFileChanger: BaseFileChanger{ diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go index ad8a74dc1..d95586074 100644 --- a/code/go/0chain.net/blobbercore/filestore/store_test.go +++ b/code/go/0chain.net/blobbercore/filestore/store_test.go @@ -251,7 +251,7 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) pathHash := encryption.Hash(test.remotePath) - hasher := GetNewCommitHasher(0) + hasher := NewCommitHasher(0) fid := &FileInputData{ Name: test.fileName, Path: test.remotePath, @@ -338,7 +338,7 @@ func TestDeletePreCommitDir(t *testing.T) { validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) pathHash := encryption.Hash(remotePath) - hasher := GetNewCommitHasher(int64(size)) + hasher := NewCommitHasher(int64(size)) fid := &FileInputData{ Name: fileName, Path: remotePath, @@ -383,7 +383,7 @@ func TestDeletePreCommitDir(t *testing.T) { fid.ValidationRoot = validationRoot fid.FixedMerkleRoot = fixedMerkleRoot - hasher = GetNewCommitHasher(int64(size)) + hasher = NewCommitHasher(int64(size)) fid.Hasher = hasher // Write file to temp location @@ -446,7 +446,7 @@ func TestStorageUploadUpdate(t *testing.T) { validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) pathHash := encryption.Hash(remotePath) - hasher := GetNewCommitHasher(int64(size)) + hasher := NewCommitHasher(int64(size)) fid := &FileInputData{ Name: fileName, Path: remotePath, @@ -834,7 +834,7 @@ func TestValidationRoot(t *testing.T) { fs, cleanUp := setupStorage(t) defer cleanUp() fPath := filepath.Join(fs.mp, randString(10)+".txt") - cH := GetNewCommitHasher(size) + cH := NewCommitHasher(size) _, err := cH.Write(thumbnailBytes) require.Nil(t, err) @@ -897,7 +897,7 @@ func generateRandomData(fPath string, size int64) (string, string, error) { } defer f.Close() - cH := GetNewCommitHasher(size) + cH := NewCommitHasher(size) _, err = cH.Write(p) if err != nil { return "", "", err @@ -938,7 +938,7 @@ func generateRandomDataAndStoreNodes(fPath string, size int64) (string, string, } defer f.Close() - cH := GetNewCommitHasher(size) + cH := NewCommitHasher(size) _, err = cH.Write(p) if err != nil { return "", "", err diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 60012ec70..6d5e19fe4 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -415,7 +415,7 @@ type CommitHasher struct { dataSize int64 } -func GetNewCommitHasher(dataSize int64) *CommitHasher { +func NewCommitHasher(dataSize int64) *CommitHasher { c := new(CommitHasher) c.md5hasher = md5.New() c.isInitialized = true From 868c861533f7a16077e4af7647fe55fc33cf6c84 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 21:08:00 +0530 Subject: [PATCH 18/98] add log for file path --- .../0chain.net/blobbercore/allocation/file_changer_upload.go | 3 +++ code/go/0chain.net/blobbercore/reference/ref.go | 1 + 2 files changed, 4 insertions(+) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 3c68a477b..342a000e4 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -8,6 +8,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "go.uber.org/zap" "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -15,6 +16,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) // swagger:model UploadFileChanger @@ -83,6 +85,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, refResult.ID = 0 // get parent id parent := filepath.Dir(nf.Path) + logging.Logger.Info("applyChange", zap.String("parent", parent), zap.String("path", nf.Path)) parentLookupHash := reference.GetReferenceLookup(nf.AllocationID, parent) err = db.Model(&reference.Ref{}).Select("id", "type").Where("lookup_hash = ?", parentLookupHash).Take(&refResult).Error if err != nil && err != gorm.ErrRecordNotFound { diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 6d412c375..d1c41a7c2 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -195,6 +195,7 @@ func NewFileRef() *Ref { // } func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { + logging.Logger.Info("mkdir", zap.String("destpath", destpath)) db := datastore.GetStore().GetTransaction(ctx) destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") fields, err := common.GetParentPaths(filepath.Dir(destpath)) From 1c129ec1f4873708795d41291c44d5f43c04f3ab Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 23:15:22 +0530 Subject: [PATCH 19/98] add check for root path --- .../allocation/file_changer_upload.go | 19 ++++----------- .../0chain.net/blobbercore/reference/ref.go | 23 ++++++++++++++++--- code/go/0chain.net/core/common/utils.go | 22 ++++++++++++++++++ 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 342a000e4..24f51c2a5 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -86,23 +86,12 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, // get parent id parent := filepath.Dir(nf.Path) logging.Logger.Info("applyChange", zap.String("parent", parent), zap.String("path", nf.Path)) - parentLookupHash := reference.GetReferenceLookup(nf.AllocationID, parent) - err = db.Model(&reference.Ref{}).Select("id", "type").Where("lookup_hash = ?", parentLookupHash).Take(&refResult).Error - if err != nil && err != gorm.ErrRecordNotFound { + // create or get parent directory + parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent) + if err != nil { return err } - // create parent directory - if refResult.ID == 0 { - parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent) - if err != nil { - return err - } - refResult.ID = parentRef.ID - } else { - if refResult.Type != reference.DIRECTORY { - return common.NewError("invalid_parameter", "parent path is not a directory") - } - } + refResult.ID = parentRef.ID newFile.ParentID = &refResult.ID collector.CreateRefRecord(newFile) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index d1c41a7c2..0c729c2d5 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -195,11 +195,25 @@ func NewFileRef() *Ref { // } func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { - logging.Logger.Info("mkdir", zap.String("destpath", destpath)) db := datastore.GetStore().GetTransaction(ctx) - destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") - fields, err := common.GetParentPaths(filepath.Dir(destpath)) + if destpath != "/" { + destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") + } + destLookupHash := GetReferenceLookup(allocationID, destpath) + var destRef Ref + err := db.Select("id", "type").Where(&Ref{LookupHash: destLookupHash}).Take(&destRef).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + if destRef.ID > 0 { + if destRef.Type != DIRECTORY { + return nil, common.NewError("invalid_dir_tree", "parent path is not a directory") + } + return &destRef, nil + } + fields, err := common.GetAllParentPaths(filepath.Dir(destpath)) if err != nil { + logging.Logger.Error("mkdir: failed to get all parent paths", zap.Error(err), zap.String("destpath", destpath)) return nil, err } @@ -226,7 +240,10 @@ func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { } } } + fields = append(fields, destpath) + parentLookupHashes = append(parentLookupHashes, destLookupHash) for i := len(parentRefs); i < len(fields); i++ { + logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Any("parentID", parentID)) var parentIDRef *int64 if parentID > 0 { parentIDRef = &parentID diff --git a/code/go/0chain.net/core/common/utils.go b/code/go/0chain.net/core/common/utils.go index d3ee7a82d..8d3250989 100644 --- a/code/go/0chain.net/core/common/utils.go +++ b/code/go/0chain.net/core/common/utils.go @@ -47,6 +47,28 @@ func GetParentPaths(fPath string) ([]string, error) { return paths[2:], nil } +func GetAllParentPaths(fPath string) ([]string, error) { + if fPath == "" { + return nil, nil + } + if fPath == "/" { + return []string{"/"}, nil + } + + fPath = filepath.Clean(fPath) + if !filepath.IsAbs(fPath) { + return nil, NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fPath)) + } + splittedPaths := strings.Split(fPath, "/") + var paths []string + fmt.Println("splittedPaths", splittedPaths) + for i := 0; i < len(splittedPaths); i++ { + subPath := strings.Join(splittedPaths[0:i], "/") + paths = append(paths, subPath) + } + return paths[2:], nil +} + // GetPathFields will return slice of fields of path. // For path /a/b/c/d/e/f.txt it will return [a, b, c, d, e, f.txt],nil func GetPathFields(p string) ([]string, error) { From 1a2edaa2ef676a1f5b026b0265ea5e8bacbded11 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 00:13:42 +0530 Subject: [PATCH 20/98] add ts --- .../0chain.net/blobbercore/allocation/file_changer_upload.go | 2 +- code/go/0chain.net/blobbercore/allocation/newdirchange.go | 2 +- code/go/0chain.net/blobbercore/reference/ref.go | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 24f51c2a5..00ffe942b 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -87,7 +87,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, parent := filepath.Dir(nf.Path) logging.Logger.Info("applyChange", zap.String("parent", parent), zap.String("path", nf.Path)) // create or get parent directory - parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent) + parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent, ts) if err != nil { return err } diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 3e6505bad..338264af9 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -18,7 +18,7 @@ type NewDir struct { func (nf *NewDir) ApplyChange(ctx context.Context, ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { - _, err := reference.Mkdir(ctx, nf.AllocationID, nf.Path) + _, err := reference.Mkdir(ctx, nf.AllocationID, nf.Path, ts) return err } diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 0c729c2d5..05cfcf015 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -194,7 +194,7 @@ func NewFileRef() *Ref { // return dirRef, nil // } -func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { +func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timestamp) (*Ref, error) { db := datastore.GetStore().GetTransaction(ctx) if destpath != "/" { destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") @@ -256,6 +256,8 @@ func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { newRef.PathLevel = i + 1 newRef.ParentID = parentIDRef newRef.LookupHash = parentLookupHashes[i] + newRef.CreatedAt = ts + newRef.UpdatedAt = ts err = db.Create(newRef).Error if err != nil { return nil, err From 4c0a6ab8d8783af29d2f413abe04ae06020dd588 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 00:53:59 +0530 Subject: [PATCH 21/98] add check for root ref --- code/go/0chain.net/blobbercore/reference/ref.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 05cfcf015..b9fc71b30 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -240,8 +240,10 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta } } } - fields = append(fields, destpath) - parentLookupHashes = append(parentLookupHashes, destLookupHash) + if destpath != "/" { + fields = append(fields, destpath) + parentLookupHashes = append(parentLookupHashes, destLookupHash) + } for i := len(parentRefs); i < len(fields); i++ { logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Any("parentID", parentID)) var parentIDRef *int64 From ce2543b2dc944e7251ee7e7e6f71cde1817520bd Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 01:25:57 +0530 Subject: [PATCH 22/98] fix version marker id --- .../blobbercore/writemarker/version_marker.go | 2 +- goose/migrations/1721021811_version_marker.sql | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/writemarker/version_marker.go b/code/go/0chain.net/blobbercore/writemarker/version_marker.go index b1dbdcfaf..757c7723c 100644 --- a/code/go/0chain.net/blobbercore/writemarker/version_marker.go +++ b/code/go/0chain.net/blobbercore/writemarker/version_marker.go @@ -12,7 +12,7 @@ import ( ) type VersionMarker struct { - ID int64 `gorm:"column:sequence;primaryKey"` + ID int64 `gorm:"column:id;primaryKey"` ClientID string `gorm:"client_id" json:"client_id"` BlobberID string `gorm:"blobber_id" json:"blobber_id"` AllocationID string `gorm:"allocation_id" json:"allocation_id"` diff --git a/goose/migrations/1721021811_version_marker.sql b/goose/migrations/1721021811_version_marker.sql index 4b035cef5..4853f693b 100644 --- a/goose/migrations/1721021811_version_marker.sql +++ b/goose/migrations/1721021811_version_marker.sql @@ -45,6 +45,17 @@ ALTER TABLE ONLY version_markers ADD CONSTRAINT version_markers_pkey PRIMARY KEY (id); +-- +-- Name: version_markers id; Type: DEFAULT; Schema: public; Owner: blobber_user +-- + +ALTER TABLE ONLY version_markers ALTER COLUMN id SET DEFAULT nextval('version_markers_id_seq'::regclass); + + +-- +-- Name: version_markers_allocation_id_idx; Type: INDEX; Schema: public; Owner: blobber_user +-- + CREATE INDEX version_markers_allocation_id_idx ON version_markers USING btree (allocation_id,version); -- +goose StatementEnd From 0a6fe4159aafea9ea17abe8ae78646f2e392e9b5 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 01:51:29 +0530 Subject: [PATCH 23/98] add file meta hash for dir --- .../handler/object_operation_handler.go | 2 +- .../0chain.net/blobbercore/reference/ref.go | 41 +------------------ 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index de8595a4a..9a0732efe 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -708,7 +708,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } if err = allocation.Repo.UpdateAllocation(ctx, allocationObj, updateMap, updateOption); err != nil { - return nil, common.NewError("allocation_write_error", "Error persisting the allocation object") + return nil, common.NewError("allocation_write_error", "Error persisting the allocation object "+err.Error()) } elapsedSaveAllocation := time.Since(startTime) - elapsedAllocation - elapsedGetLock - diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index b9fc71b30..ef7a95384 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -154,46 +154,6 @@ func NewFileRef() *Ref { } // Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child -// func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { -// var dirRef *Ref -// db := datastore.GetStore().GetTransaction(ctx) -// // cleaning path to avoid edge case issues: append '/' prefix if not added and removing suffix '/' if added -// destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") -// dirs := strings.Split(destpath, "/") - -// for i := range dirs { -// currentPath := filepath.Join("/", filepath.Join(dirs[:i+1]...)) -// ref, err := GetReference(ctx, allocationID, currentPath) -// if err == nil { -// dirRef = ref -// continue -// } - -// if !errors.Is(err, gorm.ErrRecordNotFound) { -// // unexpected sql error -// return nil, err -// } - -// // dir doesn't exists , create it -// newRef := NewDirectoryRef() -// newRef.AllocationID = allocationID -// newRef.Path = currentPath -// newRef.ParentPath = filepath.Join("/", filepath.Join(dirs[:i]...)) -// newRef.Name = dirs[i] -// newRef.Type = DIRECTORY -// newRef.PathLevel = i + 1 -// newRef.LookupHash = GetReferenceLookup(allocationID, newRef.Path) -// err = db.Create(newRef).Error -// if err != nil { -// return nil, err -// } - -// dirRef = newRef -// } - -// return dirRef, nil -// } - func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timestamp) (*Ref, error) { db := datastore.GetStore().GetTransaction(ctx) if destpath != "/" { @@ -260,6 +220,7 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta newRef.LookupHash = parentLookupHashes[i] newRef.CreatedAt = ts newRef.UpdatedAt = ts + newRef.FileMetaHash = encryption.Hash(newRef.GetFileMetaHashData()) err = db.Create(newRef).Error if err != nil { return nil, err From 67f5ee6a010894b772abce7edf6fc8fabfbbb91f Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 02:08:52 +0530 Subject: [PATCH 24/98] fix allocation update in commit --- .../blobbercore/handler/object_operation_handler.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 9a0732efe..2cc09c79c 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -691,12 +691,12 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b allocationObj.UsedSize += connectionObj.Size updateMap := map[string]interface{}{ - "used_size": allocationObj.UsedSize, - "blobber_size_used": allocationObj.BlobberSizeUsed, - "is_redeem_required": true, - "allocation_version": versionMarker.Version, - "prev_used_size": allocationObj.PrevUsedSize, - "prev_blobber_size": allocationObj.PrevBlobberSizeUsed, + "used_size": allocationObj.UsedSize, + "blobber_size_used": allocationObj.BlobberSizeUsed, + "is_redeem_required": true, + "allocation_version": versionMarker.Version, + "prev_used_size": allocationObj.PrevUsedSize, + "prev_blobber_size_used": allocationObj.PrevBlobberSizeUsed, } updateOption := func(a *allocation.Allocation) { a.IsRedeemRequired = true From 2e13ad93e572115529e732233605e9e47156b06c Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 02:30:50 +0530 Subject: [PATCH 25/98] fix file path hash in commit write --- code/go/0chain.net/blobbercore/filestore/storage.go | 6 ++---- code/go/0chain.net/blobbercore/filestore/tree_validation.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index fc7cef810..7c3fc7491 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -215,13 +215,11 @@ func (fs *FileStore) DeletePreCommitDir(allocID string) error { func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) (_ bool, err error) { now := time.Now() logging.Logger.Info("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData)) - filePathHash := encryption.Hash(fileData.Path) - tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, filePathHash, conID) - fileHash := fileData.LookupHash if fileData.IsThumbnail { fileHash = fileData.LookupHash + ThumbnailSuffix } + tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileHash, conID) preCommitPath := fs.getPreCommitPathForFile(allocID, fileHash, VERSION) @@ -292,7 +290,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) fileSize := rStat.Size() ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() - err = fileData.Hasher.Wait(ctx, conID, allocID, fileData.Name, filePathHash) + err = fileData.Hasher.Wait(ctx) if err != nil { return false, common.NewError("hasher_wait_error", err.Error()) } diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 6d5e19fe4..778e35d6d 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -490,7 +490,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil } } -func (c *CommitHasher) Wait(ctx context.Context, connID, allocID, fileName, filePathHash string) error { +func (c *CommitHasher) Wait(ctx context.Context) error { select { case <-c.doneChan: return c.hashErr From 22017d3f6e828674cd18d768e822f0f4fd6ca48b Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 12:04:34 +0530 Subject: [PATCH 26/98] get hasher in base changer --- .../0chain.net/blobbercore/allocation/file_changer_base.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index b67276ef9..5f41bc33d 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -10,7 +10,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -// swagger:model BaseFileChanger +// swagger:model BaseFileChanger // BaseFileChanger base file change processor type BaseFileChanger struct { //client side: unmarshal them from 'updateMeta'/'uploadMeta' @@ -138,6 +138,10 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mute fileInputData.LookupHash = fc.LookupHash fileInputData.ChunkSize = fc.ChunkSize fileInputData.Size = fc.Size + fileInputData.Hasher = GetHasher(fc.ConnectionID, fc.LookupHash) + if fileInputData.Hasher == nil { + return common.NewError("file_store_error", "Error getting hasher") + } _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) if err != nil { return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) From 3a4c4df698bcaba2203d1636e3568f03cd2af32b Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 12:50:04 +0530 Subject: [PATCH 27/98] check for created refs --- code/go/0chain.net/blobbercore/reference/dbCollector.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index 37c3cc808..b3905bd80 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -40,9 +40,11 @@ func (dc *dbCollector) Finalize(ctx context.Context) error { return err } } - err := db.Create(dc.createdRefs).Error - if err != nil { - return err + if len(dc.createdRefs) > 0 { + err := db.Create(dc.createdRefs).Error + if err != nil { + return err + } } return nil From dd6f54ec65eee8d1f1da041f045c9fb6c9b36424 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 24 Jul 2024 01:09:57 +0530 Subject: [PATCH 28/98] add check for dir in delete --- .../allocation/deletefilechange.go | 5 ++--- .../blobbercore/blobberhttp/response.go | 7 +++---- .../handler/file_command_delete.go | 14 +++++++++++-- .../handler/object_operation_handler.go | 4 ++-- .../blobbercore/handler/storage_handler.go | 21 ++++--------------- .../blobbercore/reference/object.go | 14 ++++--------- .../0chain.net/blobbercore/reference/ref.go | 19 ++++++++++++++++- 7 files changed, 45 insertions(+), 39 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index c683f7edd..e9f9ee0db 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -3,7 +3,6 @@ package allocation import ( "context" "encoding/json" - "path/filepath" "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -20,12 +19,12 @@ type DeleteFileChange struct { Name string `json:"name"` Path string `json:"path"` Size int64 `json:"size"` - Hash string `json:"hash"` + LookupHash string `json:"lookup_hash"` } func (nf *DeleteFileChange) ApplyChange(ctx context.Context, ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { - return reference.DeleteObject(ctx, nf.AllocationID, filepath.Clean(nf.Path), ts) + return reference.DeleteObject(ctx, nf.AllocationID, nf.LookupHash, ts) } func (nf *DeleteFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index 5b099fbce..fb64be0da 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -35,7 +35,6 @@ type RefResult struct { OffsetPath string `json:"offset_path,omitempty"` //used for pagination; index for path is created in database OffsetDate common.Timestamp `json:"offset_date,omitempty"` //used for pagination; idex for updated_at is created in database Refs *[]reference.PaginatedRef `json:"refs"` - LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` } // swagger:model RecentRefResult @@ -50,9 +49,9 @@ type ObjectPathResult struct { // swagger:model ListResult type ListResult struct { - AllocationRoot string `json:"allocation_root"` - Meta map[string]interface{} `json:"meta_data"` - Entities []map[string]interface{} `json:"list"` + AllocationVersion int64 `json:"allocation_version"` + Meta map[string]interface{} `json:"meta_data"` + Entities []map[string]interface{} `json:"list"` } // swagger:model DownloadResponse diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 5f392d938..966f3192d 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -50,13 +50,23 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request cmd.connectionID = connectionID var err error lookUpHash := reference.GetReferenceLookup(allocationObj.ID, path) - cmd.existingFileRef, err = reference.GetLimitedRefFieldsByLookupHashWith(ctx, allocationObj.ID, lookUpHash, []string{"path", "name", "size", "hash", "fixed_merkle_root"}) + cmd.existingFileRef, err = reference.GetLimitedRefFieldsByLookupHashWith(ctx, allocationObj.ID, lookUpHash, []string{"path", "name", "type", "id", "size"}) if err != nil { if errors.Is(gorm.ErrRecordNotFound, err) { return common.ErrFileWasDeleted } return common.NewError("bad_db_operation", err.Error()) } + if cmd.existingFileRef.Type == reference.DIRECTORY { + // check if directory is empty + empty, err := reference.IsDirectoryEmpty(ctx, cmd.existingFileRef.ID) + if err != nil { + return common.NewError("bad_db_operation", err.Error()) + } + if !empty { + return common.NewError("invalid_operation", "Directory is not empty") + } + } cmd.existingFileRef.LookupHash = lookUpHash return nil } @@ -82,7 +92,7 @@ func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *a connectionID := cmd.connectionID cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID, AllocationID: allocationObj.ID, Name: cmd.existingFileRef.Name, - Hash: cmd.existingFileRef.LookupHash, Path: cmd.existingFileRef.Path, Size: deleteSize} + LookupHash: cmd.existingFileRef.LookupHash, Path: cmd.existingFileRef.Path, Size: deleteSize} result := allocation.UploadResult{} result.Filename = cmd.existingFileRef.Name diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 2cc09c79c..b2e1bd4f7 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1069,7 +1069,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn return nil, common.NewError("invalid_parameters", "Invalid path") } fileRef, err := reference.GetLimitedRefFieldsByPath(ctx, connectionObj.AllocationID, path, - []string{"path", "name", "size", "hash", "validation_root", "fixed_merkle_root"}) + []string{"path", "name", "size"}) if err != nil { Logger.Error("invalid_file", zap.Error(err)) @@ -1084,7 +1084,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn allocationChange.Operation = constants.FileOperationDelete dfc := &allocation.DeleteFileChange{ConnectionID: connectionObj.ID, AllocationID: connectionObj.AllocationID, Name: fileRef.Name, - Hash: fileRef.LookupHash, Path: fileRef.Path, Size: deleteSize} + LookupHash: fileRef.LookupHash, Path: fileRef.Path, Size: deleteSize} allocation.UpdateConnectionObjSize(connectionObj.ID, allocationChange.Size) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 77518674d..aa5b98173 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -276,7 +276,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i // swagger:route GET /v1/file/list/{allocation} GetListFiles // List files. -// ListHandler is the handler to respond to list requests from clients, +// ListHandler is the handler to respond to list requests from clients, // it returns a list of files in the allocation, // along with the metadata of the files. // @@ -344,7 +344,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i // responses: // // 200: ListResult -// 400: +// 400: func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*blobberhttp.ListResult, error) { clientID := ctx.Value(constants.ContextKeyClient).(string) @@ -391,7 +391,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* if !ok { var listResult blobberhttp.ListResult - listResult.AllocationRoot = allocationObj.AllocationRoot + listResult.AllocationVersion = allocationObj.AllocationVersion if fileref == nil { fileref = &reference.Ref{Type: reference.DIRECTORY, Path: path, AllocationID: allocationID} } @@ -470,7 +470,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } var result blobberhttp.ListResult - result.AllocationRoot = allocationObj.AllocationRoot + result.AllocationVersion = allocationObj.AllocationVersion result.Meta = dirref.GetListingData(ctx) if clientID != allocationObj.OwnerID { delete(result.Meta, "path") @@ -941,24 +941,11 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb if err != nil { return nil, err } - var latestWM *writemarker.WriteMarkerEntity - if allocationObj.AllocationRoot == "" { - latestWM = nil - } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) - } - } - var refResult blobberhttp.RefResult refResult.Refs = refs refResult.TotalPages = totalPages refResult.OffsetPath = newOffsetPath refResult.OffsetDate = newOffsetDate - if latestWM != nil { - refResult.LatestWM = &latestWM.WM - } // Refs will be returned as it is and object tree will be build in client side return &refResult, nil } diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 317dea4b0..8bb14a7b2 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -3,7 +3,6 @@ package reference import ( "context" "database/sql" - "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -11,18 +10,13 @@ import ( "go.uber.org/zap" ) -func DeleteObject(ctx context.Context, allocationID, objPath string, ts common.Timestamp) error { - likePath := objPath + "/%" - if objPath == "/" { - likePath = "/%" - } - +func DeleteObject(ctx context.Context, allocationID, lookupHash string, ts common.Timestamp) error { db := datastore.GetStore().GetTransaction(ctx) - err := db.Exec("UPDATE reference_objects SET is_precommit=?,deleted_at=? WHERE allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", true, sql.NullTime{ - Time: time.Now(), + err := db.Exec("UPDATE reference_objects SET is_precommit=?,deleted_at=? WHERE lookup_hash=?", true, sql.NullTime{ + Time: common.ToTime(ts), Valid: true, - }, allocationID, "/", objPath, likePath).Error + }, lookupHash).Error if err != nil { logging.Logger.Error("delete_object_error", zap.Error(err)) return err diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index ef7a95384..55c5885f7 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -390,7 +390,7 @@ func GetSubDirsFromPath(p string) []string { func GetRefWithChildren(ctx context.Context, parentRef *Ref, allocationID, path string, offset, pageLimit int) (*Ref, error) { var refs []*Ref t := datastore.GetStore().GetTransaction(ctx) - db := t.Where(Ref{ParentPath: path, AllocationID: allocationID}) + db := t.Where(Ref{ParentID: &parentRef.ID}) err := db.Order("path"). Offset(offset). Limit(pageLimit). @@ -707,3 +707,20 @@ func GetListingFieldsMap(refEntity interface{}, tagName string) map[string]inter } return result } + +func IsDirectoryEmpty(ctx context.Context, id int64) (bool, error) { + db := datastore.GetStore().GetTransaction(ctx) + var ref Ref + err := db.Model(&Ref{}).Select("id").Where("parent_id = ?", &id).Take(&ref).Error + if err != nil { + if err == gorm.ErrRecordNotFound { + return true, nil + } + return false, err + } + if ref.ID > 0 { + return false, nil + } + + return true, nil +} From a7d405ffcdd4cdc31d063a5810e9f25cbf3a6e98 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 25 Jul 2024 18:52:55 +0530 Subject: [PATCH 29/98] fix get refs and add is empty and alloc version to ref --- .../handler/object_operation_handler.go | 21 ++-- .../blobbercore/handler/storage_handler.go | 16 ++- .../0chain.net/blobbercore/reference/ref.go | 16 ++- .../blobbercore/reference/referencepath.go | 118 +++++++++++++----- 4 files changed, 120 insertions(+), 51 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index b2e1bd4f7..00466afe8 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1353,6 +1353,13 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob var result blobberhttp.CommitResult err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed: "+err.Error()) + } + + if versionMarker.Version == allocationObj.AllocationVersion { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Version marker is same as the current version") + } elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock timeoutCtx, cancel := context.WithTimeout(ctx, 45*time.Second) @@ -1366,17 +1373,6 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob } elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem - //get allocation root and ref - rootRef, err := reference.GetLimitedRefFieldsByPath(c, allocationID, "/", []string{"hash", "file_meta_hash", "is_precommit"}) - if err != nil && err != gorm.ErrRecordNotFound { - txn.Rollback() - return nil, common.NewError("root_ref_read_error", "Error reading the root reference: "+err.Error()) - } - if err == gorm.ErrRecordNotFound { - rootRef = &reference.Ref{} - } - - Logger.Info("rollback_root_ref", zap.Any("root_ref", rootRef)) alloc, err := allocation.Repo.GetByIdAndLock(c, allocationID) Logger.Info("[rollback]Lock Allocation", zap.Bool("is_redeem_required", alloc.IsRedeemRequired), zap.String("allocation_root", alloc.AllocationRoot), zap.String("latest_wm_redeemed", alloc.LatestRedeemedWM)) if err != nil { @@ -1398,7 +1394,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob updateOption := func(a *allocation.Allocation) { a.BlobberSizeUsed = alloc.BlobberSizeUsed a.UsedSize = alloc.UsedSize - a.AllocationRoot = alloc.AllocationRoot + a.AllocationVersion = alloc.AllocationVersion a.FileMetaRoot = alloc.FileMetaRoot a.IsRedeemRequired = alloc.IsRedeemRequired } @@ -1422,7 +1418,6 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob } elapsedCommitRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem - result.AllocationRoot = allocationObj.AllocationRoot result.Success = true result.ErrorMessage = "" commitOperation := "rollback" diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index aa5b98173..1495811dc 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -142,7 +142,13 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in return nil, common.NewError("invalid_signature", "Invalid signature") } } - + if fileref.Type == reference.DIRECTORY { + fileref.IsEmpty, err = reference.IsDirectoryEmpty(ctx, fileref.ID) + if err != nil { + return nil, common.NewError("bad_db_operation", "Error checking if directory is empty. "+err.Error()) + } + } + fileref.AllocationVersion = alloc.AllocationVersion result := fileref.GetListingData(ctx) if !isOwner && !isRepairer { @@ -814,13 +820,13 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb return nil, common.NewError("invalid_parameters", "empty path and authtoken") } - var pathRef *reference.Ref + var pathRef *reference.PaginatedRef switch { case path != "": pathHash = reference.GetReferenceLookup(allocationID, path) fallthrough case pathHash != "": - pathRef, err = reference.GetReferenceByLookupHash(ctx, allocationID, pathHash) + pathRef, err = reference.GetPaginatedRefByLookupHash(ctx, pathHash) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil, common.NewError("invalid_path", "") @@ -854,7 +860,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb } if pathRef == nil { - pathRef, err = reference.GetReferenceByLookupHash(ctx, allocationID, authToken.FilePathHash) + pathRef, err = reference.GetPaginatedRefByLookupHash(ctx, authToken.FilePathHash) if err != nil { return nil, fsh.convertGormError(err) } @@ -925,7 +931,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb switch { case refType == "regular": refs, totalPages, newOffsetPath, err = reference.GetRefs( - ctx, allocationID, path, offsetPath, fileType, level, pageLimit, + ctx, allocationID, path, offsetPath, fileType, level, pageLimit, pathRef, ) case refType == "updated": diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 55c5885f7..4d62c6487 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -82,6 +82,8 @@ type Ref struct { FilestoreVersion int `gorm:"column:filestore_version" json:"-"` DataHash string `gorm:"column:data_hash" filelist:"data_hash"` DataHashSignature string `gorm:"column:data_hash_signature" filelist:"data_hash_signature"` + IsEmpty bool `gorm:"-" json:"is_empty" dirlist:"is_empty"` + AllocationVersion int64 `gorm:"-" json:"allocation_version,omitempty" dirlist:"allocation_version,omitempty" filelist:"allocation_version,omitempty"` HashToBeComputed bool `gorm:"-"` prevID int64 `gorm:"-"` } @@ -110,13 +112,11 @@ func (Ref) TableName() string { type PaginatedRef struct { //Gorm smart select fields. ID int64 `gorm:"column:id" json:"id,omitempty"` - FileID string `gorm:"file_id" json:"file_id"` Type string `gorm:"column:type" json:"type,omitempty"` AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` LookupHash string `gorm:"column:lookup_hash" json:"lookup_hash,omitempty"` Name string `gorm:"column:name" json:"name,omitempty"` Path string `gorm:"column:path" json:"path,omitempty"` - Hash string `gorm:"column:hash" json:"hash,omitempty"` NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_blocks,omitempty"` ParentPath string `gorm:"column:parent_path" json:"parent_path,omitempty"` PathLevel int `gorm:"column:level" json:"level,omitempty"` @@ -205,7 +205,7 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta parentLookupHashes = append(parentLookupHashes, destLookupHash) } for i := len(parentRefs); i < len(fields); i++ { - logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Any("parentID", parentID)) + logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Any("parentID", parentID), zap.Int("pathLevel", i+1)) var parentIDRef *int64 if parentID > 0 { parentIDRef = &parentID @@ -294,6 +294,16 @@ func GetReferenceByLookupHash(ctx context.Context, allocationID, pathHash string return ref, nil } +func GetPaginatedRefByLookupHash(ctx context.Context, pathHash string) (*PaginatedRef, error) { + ref := &PaginatedRef{} + db := datastore.GetStore().GetTransaction(ctx) + err := db.Where(&Ref{LookupHash: pathHash}).Take(ref).Error + if err != nil { + return nil, err + } + return ref, nil +} + func GetReferenceByLookupHashForDownload(ctx context.Context, allocationID, pathHash string) (*Ref, error) { ref := &Ref{} db := datastore.GetStore().GetTransaction(ctx) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 458fe9b24..b6b1470cd 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -2,6 +2,7 @@ package reference import ( "context" + "fmt" "math" "path/filepath" "strings" @@ -11,7 +12,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" - "gorm.io/gorm" ) type ReferencePath struct { @@ -238,38 +238,96 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) // Might need to consider covering index for efficient search https://blog.crunchydata.com/blog/why-covering-indexes-are-incredibly-helpful // To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs // very easily and effectively; Same case for offsetDate. -func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { - var ( - pRefs = make([]PaginatedRef, 0, pageLimit/4) - dbError error - dbQuery *gorm.DB - ) - path = filepath.Clean(path) - tx := datastore.GetStore().GetTransaction(ctx) - pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + 1 - if pathLevel == level { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND parent_path = ? and level = ?", allocationID, path, level) - if _type != "" { - dbQuery = dbQuery.Where("type = ?", _type) - } - dbQuery = dbQuery.Where("path > ?", offsetPath) - dbQuery = dbQuery.Order("path") +func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { + levelQuery := " order by path LIMIT " + fmt.Sprintf("%d", pageLimit) + if level == 0 { + level = math.MaxInt } else { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND (path=? OR path LIKE ?)", allocationID, path, path+"%") - if _type != "" { - dbQuery = dbQuery.Where("type = ?", _type) - } - if level != 0 { - dbQuery = dbQuery.Where("level = ?", level) - } - - dbQuery = dbQuery.Where("path > ?", offsetPath) + levelQuery = fmt.Sprintf(" AND level = %d order by path LIMIT %d", level, pageLimit) + } + refTypes := []string{} + if _type != "" { + refTypes = append(refTypes, _type) + } else { + refTypes = append(refTypes, "f", "d") + } + pRefs := make([]PaginatedRef, 0, pageLimit/4) + if parentRef.Type != DIRECTORY { + pRefs = append(pRefs, *parentRef) + refs = &pRefs + return + } + if offsetPath == "" || path == offsetPath { + pRefs = append(pRefs, *parentRef) + } + tx := datastore.GetStore().GetTransaction(ctx) + rows, err := tx.Raw(`WITH RECURSIVE hierarchy_cte AS ( + SELECT + id, + parent_id, + path, + level, + hierarchy.type, + size, + actual_file_size, + data_hash, + mimetype, + encrypted_key, + encrypted_key_point, + actual_file_hash_signature, + custom_meta, + num_blocks, + file_meta_hash, + parent_path + FROM + reference_objects as hierarchy + WHERE + parent_id = ? + + UNION + + SELECT + h.id, + h.parent_id, + h.path, + h.level, + h.type, + h.size, + h.actual_file_size, + h.data_hash, + h.mimetype, + h.encrypted_key, + h.encrypted_key_point, + h.actual_file_hash_signature, + h.custom_meta, + h.num_blocks, + h.file_meta_hash, + h.parent_path + FROM + hierarchy h + INNER JOIN + hierarchy_cte hc ON h.parent_id = hc.id + WHERE + hc.type='d' + AND h.level <= ? + AND h.deleted_at IS NULL + ) + SELECT * from hierarchy_cte where type IN ? AND path > ?`+levelQuery, parentRef.ID, level, refTypes, offsetPath).Rows() + if err != nil { + return + } + defer rows.Close() - dbQuery = dbQuery.Order("path") + for rows.Next() { + var ref PaginatedRef + err = tx.ScanRows(rows, &ref) + if err != nil { + return + } + ref.AllocationID = allocationID + pRefs = append(pRefs, ref) } - dbError = dbQuery.Limit(pageLimit).Find(&pRefs).Error - if dbError != nil && dbError != gorm.ErrRecordNotFound { - err = dbError + if err != nil { return } refs = &pRefs From 0321e660d3fa10c03291549aa2cee8779c09b414 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 25 Jul 2024 19:01:52 +0530 Subject: [PATCH 30/98] fix build --- code/go/0chain.net/blobbercore/convert/response_creator.go | 1 - code/go/0chain.net/blobbercore/convert/response_handler.go | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobbercore/convert/response_creator.go b/code/go/0chain.net/blobbercore/convert/response_creator.go index 828bd2bcb..039381db5 100644 --- a/code/go/0chain.net/blobbercore/convert/response_creator.go +++ b/code/go/0chain.net/blobbercore/convert/response_creator.go @@ -62,7 +62,6 @@ func ListEntitesResponseCreator(r interface{}) *blobbergrpc.ListEntitiesResponse } resp.MetaData = FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp.Meta)) - resp.AllocationRoot = httpResp.AllocationRoot return &resp } diff --git a/code/go/0chain.net/blobbercore/convert/response_handler.go b/code/go/0chain.net/blobbercore/convert/response_handler.go index 3092c6b6d..6f16e489a 100644 --- a/code/go/0chain.net/blobbercore/convert/response_handler.go +++ b/code/go/0chain.net/blobbercore/convert/response_handler.go @@ -42,9 +42,8 @@ func ListEntitesResponseHandler(resp *blobbergrpc.ListEntitiesResponse) *blobber } return &blobberhttp.ListResult{ - AllocationRoot: resp.AllocationRoot, - Meta: FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx), - Entities: entities, + Meta: FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx), + Entities: entities, } } From b1985d1fce9633a85db0ab3d2a9f9bc8daef1b6b Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 00:17:41 +0530 Subject: [PATCH 31/98] fix seq cache and add logs --- .../blobbercore/allocation/newdirchange.go | 28 ++++++++++++++++++- .../handler/object_operation_handler.go | 3 +- .../blobbercore/handler/storage_handler.go | 1 + .../0chain.net/blobbercore/reference/ref.go | 6 +++- .../1698861371_full_db_snapshot.sql | 2 +- .../migrations/1721021811_version_marker.sql | 2 +- 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 338264af9..c4fed4753 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -3,11 +3,15 @@ package allocation import ( "context" "encoding/json" + "path/filepath" + "strings" "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "gorm.io/gorm" ) type NewDir struct { @@ -18,7 +22,29 @@ type NewDir struct { func (nf *NewDir) ApplyChange(ctx context.Context, ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { - _, err := reference.Mkdir(ctx, nf.AllocationID, nf.Path, ts) + parentPath := filepath.Dir(nf.Path) + parentPathLookup := reference.GetReferenceLookup(nf.AllocationID, parentPath) + parentRef, err := reference.GetReferenceByLookupHash(ctx, nf.AllocationID, parentPathLookup) + if err != nil && err != gorm.ErrRecordNotFound { + return err + } + if parentRef.ID == 0 { + _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, ts) + } else { + parentIDRef := &parentRef.ID + newRef := reference.NewDirectoryRef() + newRef.AllocationID = nf.AllocationID + newRef.Path = nf.Path + newRef.ParentPath = parentPath + newRef.Name = filepath.Base(nf.Path) + newRef.PathLevel = len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")) + newRef.ParentID = parentIDRef + newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) + newRef.CreatedAt = ts + newRef.UpdatedAt = ts + newRef.FileMetaHash = encryption.Hash(newRef.GetFileMetaHashData()) + collector.CreateRefRecord(newRef) + } return err } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 00466afe8..d3c2a8129 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -689,6 +689,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b allocationObj.PrevUsedSize = allocationObj.UsedSize allocationObj.BlobberSizeUsed += connectionObj.Size allocationObj.UsedSize += connectionObj.Size + allocationObj.AllocationVersion = versionMarker.Version updateMap := map[string]interface{}{ "used_size": allocationObj.UsedSize, @@ -702,7 +703,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b a.IsRedeemRequired = true a.BlobberSizeUsed = allocationObj.BlobberSizeUsed a.UsedSize = allocationObj.UsedSize - a.AllocationVersion = versionMarker.Version + a.AllocationVersion = allocationObj.AllocationVersion a.PrevUsedSize = allocationObj.PrevUsedSize a.PrevBlobberSizeUsed = allocationObj.PrevBlobberSizeUsed } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 1495811dc..1f80bc6c1 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -150,6 +150,7 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in } fileref.AllocationVersion = alloc.AllocationVersion result := fileref.GetListingData(ctx) + Logger.Info("GetFileMeta", zap.Any("allocationResultVersion", result["allocation_version"]), zap.Int64("allocationVersion", alloc.AllocationVersion), zap.Any("path", result["path"])) if !isOwner && !isRepairer { var authTokenString = r.FormValue("auth_token") diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 4d62c6487..8eb790b94 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -192,11 +192,15 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta parentPath = "/" ) if len(parentRefs) > 0 { + logging.Logger.Info("mkdir:parentRefs: ", zap.Any("parentRefs", parentRefs)) parentID = parentRefs[len(parentRefs)-1].ID parentPath = parentRefs[len(parentRefs)-1].Path for i := 0; i < len(parentRefs); i++ { if parentRefs[i].Type != DIRECTORY { - return nil, common.NewError("invalid_dir_tree", "DB has invalid tree.") + return nil, common.NewError("invalid_dir_tree", "parent path is not a directory") + } + if parentRefs[i].ID == 0 { + return nil, common.NewError("invalid_dir_tree", "parent path not found") } } } diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 086f3918d..5b958b6a7 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -378,7 +378,7 @@ CREATE SEQUENCE reference_objects_id_seq INCREMENT BY 1 NO MINVALUE NO MAXVALUE - CACHE 100; + CACHE 1; ALTER TABLE reference_objects_id_seq OWNER TO blobber_user; diff --git a/goose/migrations/1721021811_version_marker.sql b/goose/migrations/1721021811_version_marker.sql index 4853f693b..dcf563a9a 100644 --- a/goose/migrations/1721021811_version_marker.sql +++ b/goose/migrations/1721021811_version_marker.sql @@ -25,7 +25,7 @@ CREATE SEQUENCE version_markers_id_seq INCREMENT BY 1 NO MINVALUE NO MAXVALUE - CACHE 100; + CACHE 1; ALTER TABLE version_markers_id_seq OWNER TO blobber_user; From 7d357759755ba6db2c7f07142599c96e51ac2c09 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 00:21:26 +0530 Subject: [PATCH 32/98] use or --- code/go/0chain.net/blobbercore/reference/ref.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 8eb790b94..7a3f8a540 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -176,14 +176,15 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta logging.Logger.Error("mkdir: failed to get all parent paths", zap.Error(err), zap.String("destpath", destpath)) return nil, err } - + tx := db.Model(&Ref{}).Select("id", "path", "type") parentLookupHashes := make([]string, 0, len(fields)) for i := 0; i < len(fields); i++ { parentLookupHashes = append(parentLookupHashes, GetReferenceLookup(allocationID, fields[i])) + tx = tx.Or(Ref{LookupHash: parentLookupHashes[i]}) } var parentRefs []*Ref - err = db.Model(&Ref{}).Select("id", "path", "type").Where("lookup_hash IN ?", parentLookupHashes).Order("path").Find(&parentRefs).Error + err = tx.Order("path").Find(&parentRefs).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } From bc2cadc86d99bb27f8f967d36218547c607ed63f Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 00:25:25 +0530 Subject: [PATCH 33/98] add check for 0 id --- code/go/0chain.net/blobbercore/reference/ref.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 7a3f8a540..655cc343c 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -214,6 +214,8 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta var parentIDRef *int64 if parentID > 0 { parentIDRef = &parentID + } else if parentPath != "/" { + return nil, common.NewError("invalid_dir_tree", "parent path not found") } newRef := NewDirectoryRef() newRef.AllocationID = allocationID From 991a7ed3c515c1672d3eeea234f5af994e5ad8e4 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 01:12:21 +0530 Subject: [PATCH 34/98] fix parent ref id --- code/go/0chain.net/blobbercore/allocation/newdirchange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index c4fed4753..0c05f0a13 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -28,7 +28,7 @@ func (nf *NewDir) ApplyChange(ctx context.Context, if err != nil && err != gorm.ErrRecordNotFound { return err } - if parentRef.ID == 0 { + if parentRef == nil || parentRef.ID == 0 { _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, ts) } else { parentIDRef := &parentRef.ID From 58e431733a2be3ff6805c7813ac9587d6441939d Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 11:54:15 +0530 Subject: [PATCH 35/98] fix get parent paths --- .../blobbercore/allocation/file_changer_upload.go | 2 +- code/go/0chain.net/blobbercore/reference/ref.go | 6 +++--- code/go/0chain.net/core/common/utils.go | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 00ffe942b..cd2766b44 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -33,7 +33,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, return common.NewError("invalid_parameter", "allocation_id is required") } - parentPath, _ := filepath.Split(nf.Path) + parentPath := filepath.Dir(nf.Path) nf.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) newFile := &reference.Ref{ ActualFileHash: nf.ActualHash, diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 655cc343c..f5cc77c21 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -82,8 +82,8 @@ type Ref struct { FilestoreVersion int `gorm:"column:filestore_version" json:"-"` DataHash string `gorm:"column:data_hash" filelist:"data_hash"` DataHashSignature string `gorm:"column:data_hash_signature" filelist:"data_hash_signature"` - IsEmpty bool `gorm:"-" json:"is_empty" dirlist:"is_empty"` - AllocationVersion int64 `gorm:"-" json:"allocation_version,omitempty" dirlist:"allocation_version,omitempty" filelist:"allocation_version,omitempty"` + IsEmpty bool `gorm:"-" dirlist:"is_empty"` + AllocationVersion int64 `gorm:"-" dirlist:"allocation_version,omitempty" filelist:"allocation_version,omitempty"` HashToBeComputed bool `gorm:"-"` prevID int64 `gorm:"-"` } @@ -171,7 +171,7 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta } return &destRef, nil } - fields, err := common.GetAllParentPaths(filepath.Dir(destpath)) + fields, err := common.GetAllParentPaths(destpath) if err != nil { logging.Logger.Error("mkdir: failed to get all parent paths", zap.Error(err), zap.String("destpath", destpath)) return nil, err diff --git a/code/go/0chain.net/core/common/utils.go b/code/go/0chain.net/core/common/utils.go index 8d3250989..7aa8cdd2b 100644 --- a/code/go/0chain.net/core/common/utils.go +++ b/code/go/0chain.net/core/common/utils.go @@ -61,12 +61,13 @@ func GetAllParentPaths(fPath string) ([]string, error) { } splittedPaths := strings.Split(fPath, "/") var paths []string - fmt.Println("splittedPaths", splittedPaths) for i := 0; i < len(splittedPaths); i++ { subPath := strings.Join(splittedPaths[0:i], "/") paths = append(paths, subPath) } - return paths[2:], nil + returnPaths := []string{"/"} + returnPaths = append(returnPaths, paths[2:]...) + return returnPaths, nil } // GetPathFields will return slice of fields of path. From 7719bd7de2d57e83cde43a21eb6b88cc6cd5d28e Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 12:44:10 +0530 Subject: [PATCH 36/98] fix dir list fields --- code/go/0chain.net/blobbercore/reference/ref.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index f5cc77c21..841300b09 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -38,7 +38,7 @@ func init() { field := refType.Field(i) dirListTag := field.Tag.Get(DIR_LIST_TAG) - if dirListTag != "" { + if dirListTag != "" && dirListTag != "is_empty" && dirListTag != "allocation_version" { dirListFields = append(dirListFields, dirListTag) } } From c62ddaa103633710dab5deaab2d17ec9179bb309 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 13:14:45 +0530 Subject: [PATCH 37/98] fix dirlist tag for allocation version --- code/go/0chain.net/blobbercore/reference/ref.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 841300b09..e0f9ace09 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -83,7 +83,7 @@ type Ref struct { DataHash string `gorm:"column:data_hash" filelist:"data_hash"` DataHashSignature string `gorm:"column:data_hash_signature" filelist:"data_hash_signature"` IsEmpty bool `gorm:"-" dirlist:"is_empty"` - AllocationVersion int64 `gorm:"-" dirlist:"allocation_version,omitempty" filelist:"allocation_version,omitempty"` + AllocationVersion int64 `gorm:"-" dirlist:"allocation_version" filelist:"allocation_version"` HashToBeComputed bool `gorm:"-"` prevID int64 `gorm:"-"` } From 79c0bf23dcb9ee296b97c1210548e094254ef123 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 13:21:55 +0530 Subject: [PATCH 38/98] add check for parent path of root --- code/go/0chain.net/blobbercore/allocation/newdirchange.go | 4 +++- code/go/0chain.net/blobbercore/reference/ref.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 0c05f0a13..b3696bca5 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -35,7 +35,9 @@ func (nf *NewDir) ApplyChange(ctx context.Context, newRef := reference.NewDirectoryRef() newRef.AllocationID = nf.AllocationID newRef.Path = nf.Path - newRef.ParentPath = parentPath + if newRef.Path != "/" { + newRef.ParentPath = parentPath + } newRef.Name = filepath.Base(nf.Path) newRef.PathLevel = len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")) newRef.ParentID = parentIDRef diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index e0f9ace09..7d51e80d5 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -220,7 +220,9 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta newRef := NewDirectoryRef() newRef.AllocationID = allocationID newRef.Path = fields[i] - newRef.ParentPath = parentPath + if newRef.Path != "/" { + newRef.ParentPath = parentPath + } newRef.Name = filepath.Base(fields[i]) newRef.PathLevel = i + 1 newRef.ParentID = parentIDRef From 12b4e25a677d9ea1c2752eaec32cc5b1f9c1ea27 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 13:48:22 +0530 Subject: [PATCH 39/98] add id to select for dir --- code/go/0chain.net/blobbercore/reference/ref.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 7d51e80d5..b34644dd2 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -42,7 +42,7 @@ func init() { dirListFields = append(dirListFields, dirListTag) } } - dirListFields = append(dirListFields, "parent_path") + dirListFields = append(dirListFields, "parent_path", "id") } type Ref struct { From 1bce447b27034bfa9e38452e625fca2ffd914162 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 14:32:00 +0530 Subject: [PATCH 40/98] set model --- code/go/0chain.net/blobbercore/reference/ref.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index b34644dd2..4ecff6be7 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -306,7 +306,7 @@ func GetReferenceByLookupHash(ctx context.Context, allocationID, pathHash string func GetPaginatedRefByLookupHash(ctx context.Context, pathHash string) (*PaginatedRef, error) { ref := &PaginatedRef{} db := datastore.GetStore().GetTransaction(ctx) - err := db.Where(&Ref{LookupHash: pathHash}).Take(ref).Error + err := db.Model(&Ref{}).Where(&Ref{LookupHash: pathHash}).Take(ref).Error if err != nil { return nil, err } From 50f6f1ca33c5f5a5988e4716a69479a59a30b198 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 14:49:18 +0530 Subject: [PATCH 41/98] fix num block column --- code/go/0chain.net/blobbercore/reference/ref.go | 2 +- code/go/0chain.net/blobbercore/reference/referencepath.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 4ecff6be7..b57e8a0b4 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -117,7 +117,7 @@ type PaginatedRef struct { //Gorm smart select fields. LookupHash string `gorm:"column:lookup_hash" json:"lookup_hash,omitempty"` Name string `gorm:"column:name" json:"name,omitempty"` Path string `gorm:"column:path" json:"path,omitempty"` - NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_blocks,omitempty"` + NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_of_blocks,omitempty"` ParentPath string `gorm:"column:parent_path" json:"parent_path,omitempty"` PathLevel int `gorm:"column:level" json:"level,omitempty"` CustomMeta string `gorm:"column:custom_meta" json:"custom_meta,omitempty"` diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index b6b1470cd..e410b6632 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -276,7 +276,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, encrypted_key_point, actual_file_hash_signature, custom_meta, - num_blocks, + num_of_blocks, file_meta_hash, parent_path FROM @@ -300,11 +300,11 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, h.encrypted_key_point, h.actual_file_hash_signature, h.custom_meta, - h.num_blocks, + h.num_of_blocks, h.file_meta_hash, h.parent_path FROM - hierarchy h + reference_objects h INNER JOIN hierarchy_cte hc ON h.parent_id = hc.id WHERE From 5a0559c32752985b7fa5581db8c0480279ee97cb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 15:23:40 +0530 Subject: [PATCH 42/98] add check for return directory --- code/go/0chain.net/blobbercore/reference/referencepath.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index e410b6632..b2919366b 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -246,10 +246,15 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, levelQuery = fmt.Sprintf(" AND level = %d order by path LIMIT %d", level, pageLimit) } refTypes := []string{} + returnDirectory := false if _type != "" { refTypes = append(refTypes, _type) + if _type == DIRECTORY { + returnDirectory = true + } } else { refTypes = append(refTypes, "f", "d") + returnDirectory = true } pRefs := make([]PaginatedRef, 0, pageLimit/4) if parentRef.Type != DIRECTORY { @@ -257,7 +262,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, refs = &pRefs return } - if offsetPath == "" || path == offsetPath { + if (offsetPath == "" || path == offsetPath) && returnDirectory { pRefs = append(pRefs, *parentRef) } tx := datastore.GetStore().GetTransaction(ctx) From 20cfb0fe04ca3f4b05c425ed0ef0c1f3aeae69b1 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 16:28:22 +0530 Subject: [PATCH 43/98] add a check for level in get refs --- .../blobbercore/reference/referencepath.go | 20 +++++++++++++------ .../1698861371_full_db_snapshot.sql | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index b2919366b..c8b72603e 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -246,23 +246,27 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, levelQuery = fmt.Sprintf(" AND level = %d order by path LIMIT %d", level, pageLimit) } refTypes := []string{} - returnDirectory := false + returnParentDirectory := false if _type != "" { refTypes = append(refTypes, _type) if _type == DIRECTORY { - returnDirectory = true + returnParentDirectory = true } } else { refTypes = append(refTypes, "f", "d") - returnDirectory = true + returnParentDirectory = true } pRefs := make([]PaginatedRef, 0, pageLimit/4) if parentRef.Type != DIRECTORY { pRefs = append(pRefs, *parentRef) refs = &pRefs return + } else if level == parentRef.PathLevel && returnParentDirectory { + pRefs = append(pRefs, *parentRef) + refs = &pRefs + return } - if (offsetPath == "" || path == offsetPath) && returnDirectory { + if (offsetPath == "" || path == offsetPath) && returnParentDirectory && level == math.MaxInt { pRefs = append(pRefs, *parentRef) } tx := datastore.GetStore().GetTransaction(ctx) @@ -279,11 +283,13 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, mimetype, encrypted_key, encrypted_key_point, + h.actual_file_hash, actual_file_hash_signature, custom_meta, num_of_blocks, file_meta_hash, - parent_path + parent_path, + updated_at FROM reference_objects as hierarchy WHERE @@ -303,11 +309,13 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, h.mimetype, h.encrypted_key, h.encrypted_key_point, + h.actual_file_hash, h.actual_file_hash_signature, h.custom_meta, h.num_of_blocks, h.file_meta_hash, - h.parent_path + h.parent_path, + h.updated_at FROM reference_objects h INNER JOIN diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 5b958b6a7..21e0dfa55 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -790,7 +790,7 @@ CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_ -- Name: idx_parent_id_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_parent_id_alloc ON reference_objects USING btree (allocation_id, parent_id) where deleted_at is NULL; +CREATE INDEX idx_parent_id_alloc ON reference_objects USING btree (parent_id) where deleted_at is NULL; -- From de124d0b1d1ab49c5234362087d018a5af7e16fa Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 16:31:32 +0530 Subject: [PATCH 44/98] order by level and path --- code/go/0chain.net/blobbercore/reference/referencepath.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index c8b72603e..0286b39ab 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -239,7 +239,7 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) // To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs // very easily and effectively; Same case for offsetDate. func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { - levelQuery := " order by path LIMIT " + fmt.Sprintf("%d", pageLimit) + levelQuery := " order by level,path LIMIT " + fmt.Sprintf("%d", pageLimit) if level == 0 { level = math.MaxInt } else { From c367f4e7a06bb180f5cebe8e48f057820d4c8afe Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 16:32:16 +0530 Subject: [PATCH 45/98] rever order by level --- code/go/0chain.net/blobbercore/reference/referencepath.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 0286b39ab..c8b72603e 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -239,7 +239,7 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) // To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs // very easily and effectively; Same case for offsetDate. func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { - levelQuery := " order by level,path LIMIT " + fmt.Sprintf("%d", pageLimit) + levelQuery := " order by path LIMIT " + fmt.Sprintf("%d", pageLimit) if level == 0 { level = math.MaxInt } else { From 2f90507e9b1da3c0eb4029a1a845bbce3950ef99 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 18:18:49 +0530 Subject: [PATCH 46/98] fix query --- code/go/0chain.net/blobbercore/reference/ref.go | 2 +- code/go/0chain.net/blobbercore/reference/referencepath.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index b57e8a0b4..081d020e1 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -132,7 +132,7 @@ type PaginatedRef struct { //Gorm smart select fields. ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"` EncryptedKey string `gorm:"column:encrypted_key" json:"encrypted_key,omitempty"` EncryptedKeyPoint string `gorm:"column:encrypted_key_point" json:"encrypted_key_point,omitempty"` - FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` + FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" json:"file_meta_hash"` CreatedAt common.Timestamp `gorm:"column:created_at" json:"created_at,omitempty"` UpdatedAt common.Timestamp `gorm:"column:updated_at" json:"updated_at,omitempty"` diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index c8b72603e..289cb7262 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -283,7 +283,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, mimetype, encrypted_key, encrypted_key_point, - h.actual_file_hash, + actual_file_hash, actual_file_hash_signature, custom_meta, num_of_blocks, From e1163d39abfb32ef8a206f0104c427bb19629567 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 19:54:12 +0530 Subject: [PATCH 47/98] remove unused fields in select --- .../blobbercore/reference/referencepath.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 289cb7262..83a5adbc2 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -277,18 +277,12 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, path, level, hierarchy.type, - size, actual_file_size, - data_hash, mimetype, encrypted_key, encrypted_key_point, actual_file_hash, - actual_file_hash_signature, custom_meta, - num_of_blocks, - file_meta_hash, - parent_path, updated_at FROM reference_objects as hierarchy @@ -303,18 +297,12 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, h.path, h.level, h.type, - h.size, h.actual_file_size, - h.data_hash, h.mimetype, h.encrypted_key, h.encrypted_key_point, h.actual_file_hash, - h.actual_file_hash_signature, h.custom_meta, - h.num_of_blocks, - h.file_meta_hash, - h.parent_path, h.updated_at FROM reference_objects h From 8b6646f3e26f5019198e389199eb14dac9cd538a Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 22:06:34 +0530 Subject: [PATCH 48/98] add check for empty dir in apply change --- .../blobbercore/allocation/common_test.go | 1 - .../allocation/deletefilechange.go | 3 +- .../allocation/file_changer_base.go | 8 +- .../allocation/renamefilechange.go | 119 ++++++---------- .../allocation/renamefilechange_main.go | 6 +- .../blobbercore/filestore/storage.go | 40 ++++++ .../0chain.net/blobbercore/filestore/store.go | 9 +- .../blobbercore/filestore/store_test.go | 14 +- .../handler/file_command_delete.go | 2 +- .../handler/file_command_update.go | 2 - .../handler/file_command_upload.go | 1 - .../handler/object_operation_handler.go | 133 +++++++++--------- .../blobbercore/handler/tests_common_test.go | 11 +- .../blobbercore/reference/object.go | 18 ++- 14 files changed, 192 insertions(+), 175 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/common_test.go b/code/go/0chain.net/blobbercore/allocation/common_test.go index 372cbaee0..b0b5c3a5a 100644 --- a/code/go/0chain.net/blobbercore/allocation/common_test.go +++ b/code/go/0chain.net/blobbercore/allocation/common_test.go @@ -24,7 +24,6 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string, Name: fileData.Name, Path: fileData.Path, FixedMerkleRoot: "", - ValidationRoot: fileData.ValidationRoot, Size: n, }, nil } diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index e9f9ee0db..d7a924e3f 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -20,11 +20,12 @@ type DeleteFileChange struct { Path string `json:"path"` Size int64 `json:"size"` LookupHash string `json:"lookup_hash"` + Type string `json:"type"` } func (nf *DeleteFileChange) ApplyChange(ctx context.Context, ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { - return reference.DeleteObject(ctx, nf.AllocationID, nf.LookupHash, ts) + return reference.DeleteObject(ctx, nf.AllocationID, nf.LookupHash, nf.Type, ts) } func (nf *DeleteFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 5f41bc33d..498a260e4 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -60,11 +60,9 @@ type BaseFileChanger struct { // swagger:model UploadResult type UploadResult struct { - Filename string `json:"filename"` - Size int64 `json:"size"` - Hash string `json:"hash"` - ValidationRoot string `json:"validation_root"` - FixedMerkleRoot string `json:"fixed_merkle_root"` + Filename string `json:"filename"` + Size int64 `json:"size"` + Hash string `json:"hash"` // UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. UploadLength int64 `json:"upload_length"` diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index fbcee3759..24dbde7ed 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -6,31 +6,36 @@ import ( "path/filepath" "sync" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" ) type RenameFileChange struct { - ConnectionID string `json:"connection_id"` - AllocationID string `json:"allocation_id"` - Path string `json:"path"` - NewName string `json:"new_name"` - Name string `json:"name"` - Type string `json:"type"` + ConnectionID string `json:"connection_id"` + AllocationID string `json:"allocation_id"` + Path string `json:"path"` + NewName string `json:"new_name"` + Name string `json:"name"` + Type string `json:"type"` + CustomMeta string `json:"custom_meta"` + MimeType string `json:"mimetype"` + newLookupHash string `json:"-"` } func (rf *RenameFileChange) DeleteTempFile() error { return nil } -func (rf *RenameFileChange) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { +func (rf *RenameFileChange) applyChange(ctx context.Context, + ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { if rf.Path == "/" { - return nil, common.NewError("invalid_operation", "cannot rename root path") + return common.NewError("invalid_operation", "cannot rename root path") } newPath := filepath.Join(filepath.Dir(rf.Path), rf.NewName) @@ -40,79 +45,33 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, rootRef *reference. } if isFilePresent { - return nil, common.NewError("invalid_reference_path", "file already exists") + return common.NewError("invalid_reference_path", "file already exists") } - affectedRef, err := rootRef.GetSrcPath(rf.Path) + ref, err := reference.GetReference(ctx, rf.AllocationID, rf.Path) if err != nil { - return nil, err + return common.NewError("invalid_reference_path", err.Error()) } - affectedRef.HashToBeComputed = true - affectedRef.Name = rf.NewName - affectedRef.Path = newPath - affectedRef.UpdatedAt = ts - if affectedRef.Type == reference.FILE { - affectedRef.IsPrecommit = true - } else { - rf.processChildren(ctx, affectedRef, ts) + deleteRef := &reference.Ref{ + ID: ref.ID, } - - parentPath := filepath.Dir(rf.Path) - fields, err := common.GetPathFields(parentPath) - if err != nil { - return nil, err - } - - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - dirRef := rootRef - - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - dirRef = child - dirRef.UpdatedAt = ts - dirRef.HashToBeComputed = true - found = true - break - } - } - - if !found { - return nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") - } - } - - found := false - for i, child := range dirRef.Children { - if child.Path == rf.Path { - dirRef.RemoveChild(i) - dirRef.AddChild(affectedRef) - found = true - break - } - } - if !found { - return nil, common.NewError("file_not_found", "File to rename not found in blobber") + collector.DeleteRefRecord(deleteRef) + ref.Name = rf.NewName + ref.Path = newPath + ref.ID = 0 + ref.LookupHash = reference.GetReferenceLookup(rf.AllocationID, newPath) + ref.UpdatedAt = ts + ref.FileMetaHash = encryption.Hash(ref.GetFileMetaHashData()) + if rf.CustomMeta != "" { + ref.CustomMeta = rf.CustomMeta } - - return rootRef, nil -} - -func (rf *RenameFileChange) processChildren(ctx context.Context, curRef *reference.Ref, ts common.Timestamp) { - for _, childRef := range curRef.Children { - childRef.UpdatedAt = ts - childRef.HashToBeComputed = true - newPath := filepath.Join(curRef.Path, childRef.Name) - childRef.UpdatePath(newPath, curRef.Path) - if childRef.Type == reference.FILE { - childRef.IsPrecommit = true - } - if childRef.Type == reference.DIRECTORY { - rf.processChildren(ctx, childRef, ts) - } + if rf.MimeType != "" { + ref.MimeType = rf.MimeType } + ref.IsPrecommit = true + collector.CreateRefRecord(ref) + rf.newLookupHash = ref.LookupHash + return nil } func (rf *RenameFileChange) Marshal() (string, error) { @@ -129,7 +88,15 @@ func (rf *RenameFileChange) Unmarshal(input string) error { } func (rf *RenameFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { - return nil + if rf.newLookupHash == "" { + return common.NewError("invalid_reference_path", "new lookup hash is empty") + } + oldFileLookupHash := reference.GetReferenceLookup(rf.AllocationID, rf.Path) + err := filestore.GetFileStore().CopyFile(rf.AllocationID, oldFileLookupHash, rf.newLookupHash) + if err != nil { + logging.Logger.Error("CommitToFileStore: CopyFile", zap.Error(err)) + } + return err } func (rf *RenameFileChange) GetPath() []string { diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go index 9c773cab7..0c91b5cb7 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go @@ -10,8 +10,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { +func (rf *RenameFileChange) ApplyChange(ctx context.Context, + ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { - return rf.applyChange(ctx, rootRef, change, allocationRoot, ts, nil) + return rf.applyChange(ctx, ts, fileIDMeta, collector) } diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 7c3fc7491..119691d5b 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -320,6 +320,46 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) return true, nil } +func (fs *FileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error { + var err error + oldObjectPath, err := fs.GetPathForFile(allocationID, oldFileLookupHash, VERSION) + if err != nil { + return common.NewError("get_file_path_error", err.Error()) + } + oldFile, err := os.Open(oldObjectPath) + if err != nil { + return common.NewError("file_open_error", err.Error()) + } + defer oldFile.Close() + stat, err := oldFile.Stat() + if err != nil { + return common.NewError("file_stat_error", err.Error()) + } + size := stat.Size() + + newObjectPath := fs.getPreCommitPathForFile(allocationID, newFileLookupHash, VERSION) + newFile, err := os.Create(newObjectPath) + if err != nil { + return common.NewError("file_create_error", err.Error()) + } + defer func() { + newFile.Close() + if err != nil { + os.Remove(newObjectPath) //nolint:errcheck + } + }() + bufSize := BufferSize + if size < int64(bufSize) { + bufSize = int(size) + } + copyBuf := make([]byte, bufSize) + _, err = io.CopyBuffer(newFile, oldFile, copyBuf) + if err != nil { + return common.NewError("file_copy_error", err.Error()) + } + return nil +} + func (fs *FileStore) GetFilePathSize(allocID, filehash, thumbHash string, version int) (int64, int64, error) { filePath, err := fs.GetPathForFile(allocID, filehash, version) diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index 5e2293da1..3cd80e210 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -30,11 +30,9 @@ type FileInputData struct { } type FileOutputData struct { - Name string - Path string - ValidationRoot string - FixedMerkleRoot string - ThumbnailHash string + Name string + Path string + ThumbnailHash string // Size written size/chunk size Size int64 // ChunkUploaded the chunk is uploaded or not. @@ -56,6 +54,7 @@ type FileStorer interface { DeleteFromFilestore(allocID, hash string, version int) error DeletePreCommitDir(allocID string) error DeleteAllocation(allocID string) + CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error // GetFileBlock Get blocks of file starting from blockNum upto numBlocks. blockNum can't be less than 1. GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadResponse, error) GetBlocksMerkleTreeForChallenge(cri *ChallengeReadBlockInput) (*ChallengeResponse, error) diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go index d95586074..66ad81b25 100644 --- a/code/go/0chain.net/blobbercore/filestore/store_test.go +++ b/code/go/0chain.net/blobbercore/filestore/store_test.go @@ -253,14 +253,12 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { pathHash := encryption.Hash(test.remotePath) hasher := NewCommitHasher(0) fid := &FileInputData{ - Name: test.fileName, - Path: test.remotePath, - ValidationRoot: validationRoot, - FixedMerkleRoot: fixedMerkleRoot, - ChunkSize: 64 * KB, - FilePathHash: pathHash, - Hasher: hasher, - Size: int64(size), + Name: test.fileName, + Path: test.remotePath, + ChunkSize: 64 * KB, + FilePathHash: pathHash, + Hasher: hasher, + Size: int64(size), } f, err := os.Open(fPath) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 966f3192d..3ee62b2eb 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -92,7 +92,7 @@ func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *a connectionID := cmd.connectionID cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID, AllocationID: allocationObj.ID, Name: cmd.existingFileRef.Name, - LookupHash: cmd.existingFileRef.LookupHash, Path: cmd.existingFileRef.Path, Size: deleteSize} + LookupHash: cmd.existingFileRef.LookupHash, Path: cmd.existingFileRef.Path, Size: deleteSize, Type: cmd.existingFileRef.Type} result := allocation.UploadResult{} result.Filename = cmd.existingFileRef.Name diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index d5579c40d..335b8ecfb 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -127,8 +127,6 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) } - result.ValidationRoot = fileOutputData.ValidationRoot - result.FixedMerkleRoot = fileOutputData.FixedMerkleRoot result.Size = fileOutputData.Size cmd.fileChanger.AllocationID = allocationObj.ID diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 3eb111fef..a7f0e9252 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -129,7 +129,6 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj return result, common.NewError("upload_error", "Failed to write file. "+err.Error()) } result.Filename = cmd.fileChanger.Filename - result.ValidationRoot = fileOutputData.ValidationRoot result.Size = fileOutputData.Size allocationSize := allocation.GetConnectionObjSize(connectionID) + cmd.fileChanger.Size diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index d3c2a8129..404781833 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -754,88 +754,89 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (interface{}, error) { - // allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) - // allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) - // allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) - // if err != nil { - // return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - // } + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) + allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + } - // if !allocationObj.CanRename() { - // return nil, common.NewError("prohibited_allocation_file_options", "Cannot rename data in this allocation.") - // } + if !allocationObj.CanRename() { + return nil, common.NewError("prohibited_allocation_file_options", "Cannot rename data in this allocation.") + } - // allocationID := allocationObj.ID + allocationID := allocationObj.ID - // clientID := ctx.Value(constants.ContextKeyClient).(string) - // _ = ctx.Value(constants.ContextKeyClientKey).(string) - // valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) - // if !valid || err != nil { - // return nil, common.NewError("invalid_signature", "Invalid signature") - // } + clientID := ctx.Value(constants.ContextKeyClient).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) + valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) + if !valid || err != nil { + return nil, common.NewError("invalid_signature", "Invalid signature") + } - // if clientID == "" { - // return nil, common.NewError("invalid_operation", "Invalid client") - // } + if clientID == "" { + return nil, common.NewError("invalid_operation", "Invalid client") + } - // new_name := r.FormValue("new_name") - // if new_name == "" { - // return nil, common.NewError("invalid_parameters", "Invalid name") - // } + new_name := r.FormValue("new_name") + if new_name == "" { + return nil, common.NewError("invalid_parameters", "Invalid name") + } - // pathHash, err := pathHashFromReq(r, allocationID) - // if err != nil { - // return nil, err - // } + pathHash, err := pathHashFromReq(r, allocationID) + if err != nil { + return nil, err + } - // if clientID == "" || allocationObj.OwnerID != clientID { - // return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - // } + if clientID == "" || allocationObj.OwnerID != clientID { + return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + } - // connectionID := r.FormValue("connection_id") - // if connectionID == "" { - // return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - // } + connectionID := r.FormValue("connection_id") + if connectionID == "" { + return nil, common.NewError("invalid_parameters", "Invalid connection id passed") + } - // connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - // if err != nil { - // return nil, common.NewError("meta_error", "Error reading metadata for connection") - // } + connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) + if err != nil { + return nil, common.NewError("meta_error", "Error reading metadata for connection") + } - // objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root", "type"}) + objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"}) - // if err != nil { - // return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - // } + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + } - // if objectRef.Path == "/" { - // return nil, common.NewError("invalid_operation", "cannot rename root path") - // } + if objectRef.Path == "/" { + return nil, common.NewError("invalid_operation", "cannot rename root path") + } - // allocationChange := &allocation.AllocationChange{} - // allocationChange.ConnectionID = connectionObj.ID - // allocationChange.Size = 0 - // allocationChange.LookupHash = pathHash - // allocationChange.Operation = constants.FileOperationRename - // dfc := &allocation.RenameFileChange{ConnectionID: connectionObj.ID, - // AllocationID: connectionObj.AllocationID, Path: objectRef.Path, Type: objectRef.Type} - // dfc.NewName = new_name - // connectionObj.AddChange(allocationChange, dfc) + if objectRef.Type != reference.FILE { + return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of directory type", objectRef.Path)) + } - // err = connectionObj.Save(ctx) - // if err != nil { - // Logger.Error("Error in writing the connection meta data", zap.Error(err)) - // return nil, common.NewError("connection_write_error", "Error writing the connection meta data") - // } + allocationChange := &allocation.AllocationChange{} + allocationChange.ConnectionID = connectionObj.ID + allocationChange.Size = 0 + allocationChange.LookupHash = pathHash + allocationChange.Operation = constants.FileOperationRename + dfc := &allocation.RenameFileChange{ConnectionID: connectionObj.ID, + AllocationID: connectionObj.AllocationID, Path: objectRef.Path, Type: objectRef.Type} + dfc.NewName = new_name + connectionObj.AddChange(allocationChange, dfc) - // result := &allocation.UploadResult{} - // result.Filename = new_name - // result.Hash = objectRef.Hash - // result.ValidationRoot = objectRef.ValidationRoot - // result.FixedMerkleRoot = objectRef.FixedMerkleRoot - // result.Size = objectRef.Size + err = connectionObj.Save(ctx) + if err != nil { + Logger.Error("Error in writing the connection meta data", zap.Error(err)) + return nil, common.NewError("connection_write_error", "Error writing the connection meta data") + } - return nil, errors.New("not implemented") + result := &allocation.UploadResult{} + result.Filename = new_name + result.Size = objectRef.Size + + return result, nil } func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (interface{}, error) { diff --git a/code/go/0chain.net/blobbercore/handler/tests_common_test.go b/code/go/0chain.net/blobbercore/handler/tests_common_test.go index 791268833..6ea548768 100644 --- a/code/go/0chain.net/blobbercore/handler/tests_common_test.go +++ b/code/go/0chain.net/blobbercore/handler/tests_common_test.go @@ -76,10 +76,9 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string, b := bytes.NewBuffer(make([]byte, 0)) n, _ := io.Copy(b, infile) return &filestore.FileOutputData{ - Name: fileData.Name, - Path: fileData.Path, - FixedMerkleRoot: "", - Size: n, + Name: fileData.Name, + Path: fileData.Path, + Size: n, }, nil } @@ -118,6 +117,10 @@ func (mfs *MockFileStore) GetTempFilePath(allocID, connID, fileName, filePathHas return "" } +func (mfs *MockFileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error { + return nil +} + func (mfs *MockFileStore) GetFileBlock(in *filestore.ReadBlockInput) (*filestore.FileDownloadResponse, error) { return &filestore.FileDownloadResponse{ Data: mockFileBlock, diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 8bb14a7b2..aa6e98df7 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -10,9 +10,23 @@ import ( "go.uber.org/zap" ) -func DeleteObject(ctx context.Context, allocationID, lookupHash string, ts common.Timestamp) error { +func DeleteObject(ctx context.Context, allocationID, lookupHash, _type string, ts common.Timestamp) error { db := datastore.GetStore().GetTransaction(ctx) - + if _type == DIRECTORY { + ref, err := GetLimitedRefFieldsByLookupHashWith(ctx, allocationID, lookupHash, []string{"id"}) + if err != nil { + logging.Logger.Error("delete_object_error", zap.Error(err)) + return err + } + isEmpty, err := IsDirectoryEmpty(ctx, ref.ID) + if err != nil { + logging.Logger.Error("delete_object_error", zap.Error(err)) + return err + } + if !isEmpty { + return common.NewError("invalid_operation", "Directory is not empty") + } + } err := db.Exec("UPDATE reference_objects SET is_precommit=?,deleted_at=? WHERE lookup_hash=?", true, sql.NullTime{ Time: common.ToTime(ts), Valid: true, From a2e49b9829eef31a6d84ecdb7a53f31b4f6ebc7a Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 26 Jul 2024 22:50:16 +0530 Subject: [PATCH 49/98] fix build and add check for update allocation --- .../blobbercore/allocation/copyfilechange.go | 112 +--------- .../allocation/file_changer_base.go | 3 +- .../allocation/file_changer_upload.go | 3 + .../allocation/renamefilechange.go | 9 + .../blobbercore/convert/response_creator.go | 30 ++- .../blobbercore/convert/response_handler.go | 10 +- .../blobbercore/filestore/storage.go | 3 + .../handler/file_command_upload.go | 3 + .../handler/object_operation_handler.go | 195 +++++++++--------- 9 files changed, 139 insertions(+), 229 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 23f69bcc1..7e0fec72f 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -3,13 +3,8 @@ package allocation import ( "context" "encoding/json" - "fmt" - "path/filepath" - "strings" "sync" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) @@ -19,115 +14,16 @@ type CopyFileChange struct { AllocationID string `json:"allocation_id"` SrcPath string `json:"path"` DestPath string `json:"dest_path"` + Type string `json:"type"` } func (rf *CopyFileChange) DeleteTempFile() error { return nil } -func (rf *CopyFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - - totalRefs, err := reference.CountRefs(ctx, rf.AllocationID) - if err != nil { - return nil, err - } - - if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs { - return nil, common.NewErrorf("max_alloc_dir_files_reached", - "maximum files and directories already reached: %v", err) - } - - srcRef, err := rootRef.GetSrcPath(rf.SrcPath) - if err != nil { - return nil, err - } - - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - - dirRef := rootRef - fields, err := common.GetPathFields(rf.DestPath) - if err != nil { - return nil, err - } - - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - if child.Type == reference.DIRECTORY { - child.HashToBeComputed = true - dirRef = child - dirRef.UpdatedAt = ts - found = true - } else { - return nil, common.NewError("invalid_path", - fmt.Sprintf("%s is of file type", child.Path)) - } - } - } - - if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir { - return nil, common.NewErrorf("max_objects_in_dir_reached", - "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir) - } - - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = rf.AllocationID - newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/")) - newRef.ParentPath = filepath.Join("/", strings.Join(fields[:i], "/")) - newRef.Name = fields[i] - newRef.HashToBeComputed = true - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - dirRef.AddChild(newRef) - dirRef = newRef - } - } - - _, err = rf.processCopyRefs(ctx, srcRef, dirRef, allocationRoot, ts, fileIDMeta) - if err != nil { - return nil, err - } - - return rootRef, err -} - -func (rf *CopyFileChange) processCopyRefs( - ctx context.Context, srcRef, destRef *reference.Ref, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string, -) ( - fileRefs []*reference.Ref, err error, -) { - - newRef := *srcRef - newRef.ID = 0 - newRef.Path = filepath.Join(destRef.Path, srcRef.Name) - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.ParentPath = destRef.Path - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - newRef.HashToBeComputed = true - destRef.AddChild(&newRef) - if newRef.Type == reference.DIRECTORY { - for _, childRef := range srcRef.Children { - fRefs, err := rf.processCopyRefs(ctx, childRef, &newRef, allocationRoot, ts, fileIDMeta) - if err != nil { - return nil, err - } - fileRefs = append(fileRefs, fRefs...) - } - } else { - fileRefs = append(fileRefs, &newRef) - } - - return +func (rf *CopyFileChange) ApplyChange(ctx context.Context, + ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + return common.NewError("not_implemented", "not implemented") } func (rf *CopyFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 498a260e4..e39c1d671 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -55,7 +55,8 @@ type BaseFileChanger struct { ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of CompactMerkleTree ChunkHash string `json:"chunk_hash,omitempty"` UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to - LookupHash string `json:"-"` // hash of path + CanUpdate bool `json:"can_update"` // can file be updated or not + LookupHash string `json:"-"` // hash of allocationID+path } // swagger:model UploadResult diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index cd2766b44..6742d6826 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -77,6 +77,9 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, return err } if refResult.ID > 0 { + if !nf.CanUpdate { + return common.NewError("prohibited_allocation_file_options", "Cannot update data in this allocation.") + } deleteRecord := &reference.Ref{ ID: refResult.ID, } diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index 24dbde7ed..cce87cf36 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -52,6 +52,15 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, if err != nil { return common.NewError("invalid_reference_path", err.Error()) } + if ref.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, ref.ID) + if err != nil { + return common.NewError("invalid_reference_path", err.Error()) + } + if !isEmpty { + return common.NewError("invalid_reference_path", "Directory is not empty") + } + } deleteRef := &reference.Ref{ ID: ref.ID, } diff --git a/code/go/0chain.net/blobbercore/convert/response_creator.go b/code/go/0chain.net/blobbercore/convert/response_creator.go index 039381db5..7e8dbbfdc 100644 --- a/code/go/0chain.net/blobbercore/convert/response_creator.go +++ b/code/go/0chain.net/blobbercore/convert/response_creator.go @@ -156,12 +156,10 @@ func CopyObjectResponseCreator(r interface{}) *blobbergrpc.CopyObjectResponse { httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.CopyObjectResponse{ - Filename: httpResp.Filename, - Size: httpResp.Size, - ValidationRoot: httpResp.ValidationRoot, - FixedMerkleRoot: httpResp.FixedMerkleRoot, - UploadLength: httpResp.UploadLength, - UploadOffset: httpResp.UploadOffset, + Filename: httpResp.Filename, + Size: httpResp.Size, + UploadLength: httpResp.UploadLength, + UploadOffset: httpResp.UploadOffset, } } @@ -172,12 +170,10 @@ func RenameObjectResponseCreator(r interface{}) *blobbergrpc.RenameObjectRespons httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.RenameObjectResponse{ - Filename: httpResp.Filename, - Size: httpResp.Size, - ValidationRoot: httpResp.ValidationRoot, - FixedMerkleRoot: httpResp.FixedMerkleRoot, - UploadLength: httpResp.UploadLength, - UploadOffset: httpResp.UploadOffset, + Filename: httpResp.Filename, + Size: httpResp.Size, + UploadLength: httpResp.UploadLength, + UploadOffset: httpResp.UploadOffset, } } @@ -210,11 +206,9 @@ func UploadFileResponseCreator(r interface{}) *blobbergrpc.UploadFileResponse { httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.UploadFileResponse{ - Filename: httpResp.Filename, - Size: httpResp.Size, - ValidationRoot: httpResp.ValidationRoot, - FixedMerkleRoot: httpResp.FixedMerkleRoot, - UploadLength: httpResp.UploadLength, - UploadOffset: httpResp.UploadOffset, + Filename: httpResp.Filename, + Size: httpResp.Size, + UploadLength: httpResp.UploadLength, + UploadOffset: httpResp.UploadOffset, } } diff --git a/code/go/0chain.net/blobbercore/convert/response_handler.go b/code/go/0chain.net/blobbercore/convert/response_handler.go index 6f16e489a..8fb3374df 100644 --- a/code/go/0chain.net/blobbercore/convert/response_handler.go +++ b/code/go/0chain.net/blobbercore/convert/response_handler.go @@ -118,11 +118,9 @@ func GetCommitMetaTxnHandlerResponse(response *blobbergrpc.CommitMetaTxnResponse func CopyObjectResponseHandler(copyObjectResponse *blobbergrpc.CopyObjectResponse) *allocation.UploadResult { return &allocation.UploadResult{ - Filename: copyObjectResponse.Filename, - Size: copyObjectResponse.Size, - ValidationRoot: copyObjectResponse.ValidationRoot, - FixedMerkleRoot: copyObjectResponse.FixedMerkleRoot, - UploadLength: copyObjectResponse.UploadLength, - UploadOffset: copyObjectResponse.UploadOffset, + Filename: copyObjectResponse.Filename, + Size: copyObjectResponse.Size, + UploadLength: copyObjectResponse.UploadLength, + UploadOffset: copyObjectResponse.UploadOffset, } } diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 119691d5b..ec3bcbfa4 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -321,6 +321,9 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) } func (fs *FileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error { + if oldFileLookupHash == newFileLookupHash { + return nil + } var err error oldObjectPath, err := fs.GetPathForFile(allocationID, oldFileLookupHash, VERSION) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index a7f0e9252..3c4330d48 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -96,6 +96,9 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request if fileChanger.ChunkSize <= 0 { fileChanger.ChunkSize = fileref.CHUNK_SIZE } + if allocationObj.CanUpdate() { + fileChanger.CanUpdate = true + } origfile, _, err := req.FormFile(UploadFile) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 404781833..46e08a6da 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -88,14 +88,6 @@ func readPreRedeem( return } -func checkPendingMarkers(ctx context.Context, allocationID string) error { - pending := writemarker.CheckProcessingMarker(allocationID) - if pending { - return common.NewError("pending_markers", "previous marker is still pending to be redeemed") - } - return nil -} - func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMarker *writemarker.WriteMarker, payerID string) (err error) { // check out read pool tokens if read_price > 0 var ( @@ -813,7 +805,13 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i } if objectRef.Type != reference.FILE { - return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of directory type", objectRef.Path)) + isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID) + if err != nil { + return nil, common.NewError("invalid_operation", "Error checking if directory is empty") + } + if !isEmpty { + return nil, common.NewError("invalid_operation", "Directory is not empty") + } } allocationChange := &allocation.AllocationChange{} @@ -841,111 +839,116 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (interface{}, error) { - // allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) - // allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) - // allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) - // if err != nil { - // return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - // } + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) + allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + } - // if !allocationObj.CanCopy() { - // return nil, common.NewError("prohibited_allocation_file_options", "Cannot copy data from this allocation.") - // } + if !allocationObj.CanCopy() { + return nil, common.NewError("prohibited_allocation_file_options", "Cannot copy data from this allocation.") + } - // valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) - // if !valid || err != nil { - // return nil, common.NewError("invalid_signature", "Invalid signature") - // } + valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) + if !valid || err != nil { + return nil, common.NewError("invalid_signature", "Invalid signature") + } - // clientID := ctx.Value(constants.ContextKeyClient).(string) - // _ = ctx.Value(constants.ContextKeyClientKey).(string) + clientID := ctx.Value(constants.ContextKeyClient).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) - // allocationID := allocationObj.ID + allocationID := allocationObj.ID - // if clientID == "" { - // return nil, common.NewError("invalid_operation", "Invalid client") - // } + if clientID == "" { + return nil, common.NewError("invalid_operation", "Invalid client") + } - // destPath := r.FormValue("dest") - // if destPath == "" { - // return nil, common.NewError("invalid_parameters", "Invalid destination for operation") - // } + destPath := r.FormValue("dest") + if destPath == "" { + return nil, common.NewError("invalid_parameters", "Invalid destination for operation") + } - // pathHash, err := pathHashFromReq(r, allocationID) - // if err != nil { - // return nil, err - // } + pathHash, err := pathHashFromReq(r, allocationID) + if err != nil { + return nil, err + } - // if clientID == "" || allocationObj.OwnerID != clientID { - // return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - // } + if clientID == "" || allocationObj.OwnerID != clientID { + return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + } - // connectionID := r.FormValue("connection_id") - // if connectionID == "" { - // return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - // } + connectionID := r.FormValue("connection_id") + if connectionID == "" { + return nil, common.NewError("invalid_parameters", "Invalid connection id passed") + } - // connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - // if err != nil { - // return nil, common.NewError("meta_error", "Error reading metadata for connection") - // } + connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) + if err != nil { + return nil, common.NewError("meta_error", "Error reading metadata for connection") + } - // objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) + objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "type"}) - // if err != nil { - // return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - // } - // if objectRef.ParentPath == destPath || objectRef.Path == destPath { - // return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot copy to the same parent directory.") - // } - // newPath := filepath.Join(destPath, objectRef.Name) - // paths, err := common.GetParentPaths(newPath) - // if err != nil { - // return nil, err - // } + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + } + if objectRef.ParentPath == destPath || objectRef.Path == destPath { + return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot copy to the same parent directory.") + } + if objectRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID) + if err != nil { + return nil, common.NewError("invalid_operation", "Error checking if directory is empty") + } + if !isEmpty { + return nil, common.NewError("invalid_operation", "Directory is not empty") + } + } + newPath := filepath.Join(destPath, objectRef.Name) + paths, err := common.GetParentPaths(newPath) + if err != nil { + return nil, err + } - // paths = append(paths, newPath) + paths = append(paths, newPath) - // refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) - // if err != nil { - // Logger.Error("Database error", zap.Error(err)) - // return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) - // } + refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) + if err != nil { + Logger.Error("Database error", zap.Error(err)) + return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) + } - // for _, ref := range refs { - // switch ref.Path { - // case newPath: - // return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") - // default: - // if ref.Type == reference.FILE { - // return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) - // } - // } - // } + for _, ref := range refs { + switch ref.Path { + case newPath: + return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") + default: + if ref.Type == reference.FILE { + return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) + } + } + } - // allocationChange := &allocation.AllocationChange{} - // allocationChange.ConnectionID = connectionObj.ID - // allocationChange.Size = objectRef.Size - // allocationChange.LookupHash = pathHash - // allocationChange.Operation = constants.FileOperationCopy - // dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ID, - // AllocationID: connectionObj.AllocationID, DestPath: destPath} - // dfc.SrcPath = objectRef.Path - // allocation.UpdateConnectionObjSize(connectionID, allocationChange.Size) - // connectionObj.AddChange(allocationChange, dfc) + allocationChange := &allocation.AllocationChange{} + allocationChange.ConnectionID = connectionObj.ID + allocationChange.Size = objectRef.Size + allocationChange.LookupHash = pathHash + allocationChange.Operation = constants.FileOperationCopy + dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ID, + AllocationID: connectionObj.AllocationID, DestPath: destPath, Type: objectRef.Type, SrcPath: objectRef.Path} + allocation.UpdateConnectionObjSize(connectionID, allocationChange.Size) + connectionObj.AddChange(allocationChange, dfc) - // err = connectionObj.Save(ctx) - // if err != nil { - // Logger.Error("Error in writing the connection meta data", zap.Error(err)) - // return nil, common.NewError("connection_write_error", "Error writing the connection meta data") - // } + err = connectionObj.Save(ctx) + if err != nil { + Logger.Error("Error in writing the connection meta data", zap.Error(err)) + return nil, common.NewError("connection_write_error", "Error writing the connection meta data") + } - // result := &allocation.UploadResult{} - // result.Filename = objectRef.Name - // result.Hash = objectRef.Hash - // result.ValidationRoot = objectRef.ValidationRoot - // result.FixedMerkleRoot = objectRef.FixedMerkleRoot - // result.Size = objectRef.Size + result := &allocation.UploadResult{} + result.Filename = objectRef.Name + result.Size = objectRef.Size return nil, errors.New("not implemented") } From 1bb9f00082fa0257f0c1aab2de1e628213d681c8 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 12:24:25 +0530 Subject: [PATCH 50/98] add move,copy and rename --- .../allocation/allocationchange.go | 12 +- .../blobbercore/allocation/copyfilechange.go | 60 ++++- .../allocation/file_changer_upload.go | 1 - .../blobbercore/allocation/movefilechange.go | 155 ++++-------- .../blobbercore/allocation/newdirchange.go | 8 + .../allocation/renamefilechange.go | 4 + .../blobbercore/filestore/storage.go | 24 ++ .../handler/object_operation_handler.go | 231 +++++++++--------- 8 files changed, 264 insertions(+), 231 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index ecbc299a6..47014087c 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -236,14 +236,14 @@ func (cc *AllocationChangeCollector) ComputeProperties() { // acp = new(UpdateFileChanger) case constants.FileOperationDelete: acp = new(DeleteFileChange) - // case constants.FileOperationRename: - // acp = new(RenameFileChange) - // case constants.FileOperationCopy: - // acp = new(CopyFileChange) + case constants.FileOperationRename: + acp = new(RenameFileChange) + case constants.FileOperationCopy: + acp = new(CopyFileChange) case constants.FileOperationCreateDir: acp = new(NewDir) - // case constants.FileOperationMove: - // acp = new(MoveFileChange) + case constants.FileOperationMove: + acp = new(MoveFileChange) } if acp == nil { diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 7e0fec72f..4389d19d9 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -3,10 +3,14 @@ package allocation import ( "context" "encoding/json" + "path/filepath" + "strings" "sync" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) type CopyFileChange struct { @@ -15,6 +19,7 @@ type CopyFileChange struct { SrcPath string `json:"path"` DestPath string `json:"dest_path"` Type string `json:"type"` + CustomMeta string `json:"custom_meta"` } func (rf *CopyFileChange) DeleteTempFile() error { @@ -23,7 +28,53 @@ func (rf *CopyFileChange) DeleteTempFile() error { func (rf *CopyFileChange) ApplyChange(ctx context.Context, ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { - return common.NewError("not_implemented", "not implemented") + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) + srcRef, err := reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash) + if err != nil { + return err + } + exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath) + if err != nil { + return err + } + if exist { + return common.NewError("invalid_reference_path", "file already exists") + } + + rf.Type = srcRef.Type + if srcRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, srcRef.ID) + if err != nil { + return err + } + if !isEmpty { + return common.NewError("invalid_reference_path", "directory is not empty") + } + } + + parentDir, err := reference.Mkdir(ctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts) + if err != nil { + return err + } + + srcRef.ID = 0 + srcRef.ParentID = &parentDir.ID + srcRef.Path = rf.DestPath + srcRef.LookupHash = destLookUpHash + srcRef.CreatedAt = ts + srcRef.UpdatedAt = ts + srcRef.ParentPath = filepath.Dir(rf.DestPath) + srcRef.Name = filepath.Base(rf.DestPath) + srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/")) + srcRef.FileMetaHash = encryption.Hash(srcRef.GetFileHashData()) + if rf.CustomMeta != "" { + srcRef.CustomMeta = rf.CustomMeta + } + srcRef.IsPrecommit = true + collector.CreateRefRecord(srcRef) + + return nil } func (rf *CopyFileChange) Marshal() (string, error) { @@ -40,7 +91,12 @@ func (rf *CopyFileChange) Unmarshal(input string) error { } func (rf *CopyFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { - return nil + if rf.Type == reference.DIRECTORY { + return nil + } + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) + return filestore.GetFileStore().CopyFile(rf.AllocationID, srcLookUpHash, destLookUpHash) } func (rf *CopyFileChange) GetPath() []string { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 6742d6826..115a5875d 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -56,7 +56,6 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, ChunkSize: nf.ChunkSize, CreatedAt: ts, UpdatedAt: ts, - HashToBeComputed: true, IsPrecommit: true, LookupHash: nf.LookupHash, DataHash: nf.DataHash, diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange.go b/code/go/0chain.net/blobbercore/allocation/movefilechange.go index 5bdfee033..b926b1a11 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange.go @@ -3,14 +3,14 @@ package allocation import ( "context" "encoding/json" - "fmt" "path/filepath" "strings" "sync" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) type MoveFileChange struct { @@ -18,134 +18,64 @@ type MoveFileChange struct { AllocationID string `json:"allocation_id"` SrcPath string `json:"path"` DestPath string `json:"dest_path"` + Type string `json:"type"` } func (rf *MoveFileChange) DeleteTempFile() error { return nil } -func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { +func (rf *MoveFileChange) ApplyChange(cctx context.Context, + ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) - srcRef, err := rootRef.GetSrcPath(rf.SrcPath) + srcRef, err := reference.GetReferenceByLookupHash(cctx, rf.AllocationID, srcLookUpHash) if err != nil { - return nil, err + return err } - - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - - srcParentPath, srcFileName := filepath.Split(rf.SrcPath) - srcFields, err := common.GetPathFields(srcParentPath) + exist, err := reference.IsRefExist(cctx, rf.AllocationID, rf.DestPath) if err != nil { - return nil, err + return err } - dirRef := rootRef - for i := 0; i < len(srcFields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == srcFields[i] { - dirRef = child - found = true - dirRef.HashToBeComputed = true - break - } - } - if !found { - return nil, common.NewError("invalid_reference_path", - fmt.Sprintf("path %s does not exist", strings.Join(srcFields[:i+1], "/"))) - } - } - - var removed bool - for i, child := range dirRef.Children { - if child.Name == srcFileName { - dirRef.RemoveChild(i) - removed = true - break - } + if exist { + return common.NewError("invalid_reference_path", "file already exists") } - if !removed { - return nil, common.NewError("incomplete_move", - "move operation rejected as it cannot be completed") - } - - dirRef = rootRef - fields, err := common.GetPathFields(rf.DestPath) - if err != nil { - return nil, err - } - - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - if child.Type == reference.DIRECTORY { - child.HashToBeComputed = true - dirRef = child - dirRef.UpdatedAt = ts - found = true - } else { - return nil, common.NewError("invalid_path", - fmt.Sprintf("%s is of file type", child.Path)) - } - } - } - if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir { - return nil, common.NewErrorf("max_objects_in_dir_reached", - "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir) + if srcRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(cctx, srcRef.ID) + if err != nil { + return err } - - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = rf.AllocationID - newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/")) - newRef.ParentPath = filepath.Join("/", strings.Join(fields[:i], "/")) - newRef.Name = fields[i] - newRef.HashToBeComputed = true - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - dirRef.AddChild(newRef) - dirRef = newRef + if !isEmpty { + return common.NewError("invalid_reference_path", "directory is not empty") } } - fileRefs := rf.processMoveRefs(ctx, srcRef, dirRef, allocationRoot, ts, true) + rf.Type = srcRef.Type - for _, fileRef := range fileRefs { - fileRef.IsPrecommit = true + parentDir, err := reference.Mkdir(cctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts) + if err != nil { + return err } - return rootRef, nil -} - -func (rf *MoveFileChange) processMoveRefs( - ctx context.Context, srcRef, destRef *reference.Ref, - allocationRoot string, ts common.Timestamp, toAdd bool) (fileRefs []*reference.Ref) { - if srcRef.Type == reference.DIRECTORY { - srcRef.Path = filepath.Join(destRef.Path, srcRef.Name) - srcRef.ParentPath = destRef.Path - srcRef.UpdatedAt = ts - srcRef.HashToBeComputed = true - if toAdd { - destRef.AddChild(srcRef) - } - for _, childRef := range srcRef.Children { - fileRefs = append(fileRefs, rf.processMoveRefs(ctx, childRef, srcRef, allocationRoot, ts, false)...) - } - } else if srcRef.Type == reference.FILE { - srcRef.ParentPath = destRef.Path - srcRef.Path = filepath.Join(destRef.Path, srcRef.Name) - srcRef.UpdatedAt = ts - srcRef.HashToBeComputed = true - if toAdd { - destRef.AddChild(srcRef) - } - fileRefs = append(fileRefs, srcRef) + deleteRef := &reference.Ref{ + ID: srcRef.ID, } + collector.DeleteRefRecord(deleteRef) + + srcRef.ID = 0 + srcRef.ParentID = &parentDir.ID + srcRef.Path = rf.DestPath + srcRef.ParentPath = filepath.Dir(rf.DestPath) + srcRef.Name = filepath.Base(rf.DestPath) + srcRef.LookupHash = destLookUpHash + srcRef.CreatedAt = ts + srcRef.UpdatedAt = ts + srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/")) + srcRef.FileMetaHash = encryption.Hash(srcRef.GetFileHashData()) + collector.CreateRefRecord(srcRef) - return - + return nil } func (rf *MoveFileChange) Marshal() (string, error) { @@ -162,7 +92,12 @@ func (rf *MoveFileChange) Unmarshal(input string) error { } func (rf *MoveFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { - return nil + if rf.Type == reference.DIRECTORY { + return nil + } + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) + return filestore.GetFileStore().CopyFile(rf.AllocationID, srcLookUpHash, destLookUpHash) } func (rf *MoveFileChange) GetPath() []string { diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index b3696bca5..69d31616f 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -18,6 +18,8 @@ type NewDir struct { ConnectionID string `json:"connection_id" validation:"required"` Path string `json:"filepath" validation:"required"` AllocationID string `json:"allocation_id"` + CustomMeta string `json:"custom_meta,omitempty"` + MimeType string `json:"mimetype,omitempty"` } func (nf *NewDir) ApplyChange(ctx context.Context, @@ -45,6 +47,12 @@ func (nf *NewDir) ApplyChange(ctx context.Context, newRef.CreatedAt = ts newRef.UpdatedAt = ts newRef.FileMetaHash = encryption.Hash(newRef.GetFileMetaHashData()) + if nf.CustomMeta != "" { + newRef.CustomMeta = nf.CustomMeta + } + if nf.MimeType != "" { + newRef.MimeType = nf.MimeType + } collector.CreateRefRecord(newRef) } return err diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index cce87cf36..c197bd082 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -61,6 +61,7 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, return common.NewError("invalid_reference_path", "Directory is not empty") } } + rf.Type = ref.Type deleteRef := &reference.Ref{ ID: ref.ID, } @@ -97,6 +98,9 @@ func (rf *RenameFileChange) Unmarshal(input string) error { } func (rf *RenameFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { + if rf.Type == reference.DIRECTORY { + return nil + } if rf.newLookupHash == "" { return common.NewError("invalid_reference_path", "new lookup hash is empty") } diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index ec3bcbfa4..e432d9c6d 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -360,6 +360,30 @@ func (fs *FileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash if err != nil { return common.NewError("file_copy_error", err.Error()) } + // copy thumbnail if exists + oldThumbPath := fs.getPreCommitPathForFile(allocationID, oldFileLookupHash+ThumbnailSuffix, VERSION) + if _, err := os.Stat(oldThumbPath); err == nil { + newThumbPath := fs.getPreCommitPathForFile(allocationID, newFileLookupHash+ThumbnailSuffix, VERSION) + oldThumbFile, err := os.Open(oldThumbPath) + if err != nil { + return common.NewError("file_open_error", err.Error()) + } + defer oldThumbFile.Close() + newThumbFile, err := os.Create(newThumbPath) + if err != nil { + return common.NewError("file_create_error", err.Error()) + } + defer func() { + newThumbFile.Close() + if err != nil { + os.Remove(newThumbPath) //nolint:errcheck + } + }() + _, err = io.Copy(newThumbFile, oldThumbFile) + if err != nil { + return common.NewError("file_copy_error", err.Error()) + } + } return nil } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 46e08a6da..ab0f97950 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -949,122 +949,129 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int result := &allocation.UploadResult{} result.Filename = objectRef.Name result.Size = objectRef.Size - return nil, errors.New("not implemented") + return result, nil } func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (interface{}, error) { - // allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) - // allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) - // allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) - // if err != nil { - // return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) - // } - - // if !allocationObj.CanMove() { - // return nil, common.NewError("prohibited_allocation_file_options", "Cannot move data in this allocation.") - // } - - // valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) - // if !valid || err != nil { - // return nil, common.NewError("invalid_signature", "Invalid signature") - // } - - // clientID := ctx.Value(constants.ContextKeyClient).(string) - // _ = ctx.Value(constants.ContextKeyClientKey).(string) - - // allocationID := allocationObj.ID - - // if clientID == "" { - // return nil, common.NewError("invalid_operation", "Invalid client") - // } - - // destPath := r.FormValue("dest") - // if destPath == "" { - // return nil, common.NewError("invalid_parameters", "Invalid destination for operation") - // } - - // pathHash, err := pathHashFromReq(r, allocationID) - // if err != nil { - // return nil, err - // } - - // if clientID == "" || allocationObj.OwnerID != clientID { - // return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") - // } - - // connectionID := r.FormValue("connection_id") - // if connectionID == "" { - // return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - // } - - // connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) - // if err != nil { - // return nil, common.NewError("meta_error", "Error reading metadata for connection") - // } - - // objectRef, err := reference.GetLimitedRefFieldsByLookupHash( - // ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) - - // if err != nil { - // return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) - // } - - // if objectRef.ParentPath == destPath { - // return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot move to the same parent directory.") - // } - // newPath := filepath.Join(destPath, objectRef.Name) - // paths, err := common.GetParentPaths(newPath) - // if err != nil { - // return nil, err - // } - - // paths = append(paths, newPath) - - // refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) - // if err != nil { - // Logger.Error("Database error", zap.Error(err)) - // return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) - // } - - // for _, ref := range refs { - // switch ref.Path { - // case newPath: - // return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") - // default: - // if ref.Type == reference.FILE { - // return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) - // } - // } - // } - - // allocationChange := &allocation.AllocationChange{} - // allocationChange.ConnectionID = connectionObj.ID - // allocationChange.Size = 0 - // allocationChange.LookupHash = pathHash - // allocationChange.Operation = constants.FileOperationMove - // dfc := &allocation.MoveFileChange{ - // ConnectionID: connectionObj.ID, - // AllocationID: connectionObj.AllocationID, - // SrcPath: objectRef.Path, - // DestPath: destPath, - // } - // dfc.SrcPath = objectRef.Path - // connectionObj.AddChange(allocationChange, dfc) - - // err = connectionObj.Save(ctx) - // if err != nil { - // Logger.Error("Error in writing the connection meta data", zap.Error(err)) - // return nil, common.NewError("connection_write_error", "Error writing the connection meta data") - // } - - // result := &allocation.UploadResult{} - // result.Filename = objectRef.Name - // result.Hash = objectRef.Hash - // result.ValidationRoot = objectRef.ValidationRoot - // result.FixedMerkleRoot = objectRef.FixedMerkleRoot - // result.Size = objectRef.Size - return nil, errors.New("not implemented") + allocationId := ctx.Value(constants.ContextKeyAllocationID).(string) + allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) + allocationObj, err := fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + } + + if !allocationObj.CanMove() { + return nil, common.NewError("prohibited_allocation_file_options", "Cannot move data in this allocation.") + } + + valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), r.Header.Get(common.ClientSignatureHeaderV2), allocationObj.OwnerPublicKey) + if !valid || err != nil { + return nil, common.NewError("invalid_signature", "Invalid signature") + } + + clientID := ctx.Value(constants.ContextKeyClient).(string) + _ = ctx.Value(constants.ContextKeyClientKey).(string) + + allocationID := allocationObj.ID + + if clientID == "" { + return nil, common.NewError("invalid_operation", "Invalid client") + } + + destPath := r.FormValue("dest") + if destPath == "" { + return nil, common.NewError("invalid_parameters", "Invalid destination for operation") + } + + pathHash, err := pathHashFromReq(r, allocationID) + if err != nil { + return nil, err + } + + if clientID == "" || allocationObj.OwnerID != clientID { + return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + } + + connectionID := r.FormValue("connection_id") + if connectionID == "" { + return nil, common.NewError("invalid_parameters", "Invalid connection id passed") + } + + connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) + if err != nil { + return nil, common.NewError("meta_error", "Error reading metadata for connection") + } + + objectRef, err := reference.GetLimitedRefFieldsByLookupHash( + ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"}) + + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + } + + if objectRef.ParentPath == destPath { + return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot move to the same parent directory.") + } + if objectRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID) + if err != nil { + return nil, common.NewError("invalid_operation", "Error checking if directory is empty") + } + if !isEmpty { + return nil, common.NewError("invalid_operation", "Directory is not empty") + } + } + newPath := filepath.Join(destPath, objectRef.Name) + paths, err := common.GetParentPaths(newPath) + if err != nil { + return nil, err + } + + paths = append(paths, newPath) + + refs, err := reference.GetRefsTypeFromPaths(ctx, allocationID, paths) + if err != nil { + Logger.Error("Database error", zap.Error(err)) + return nil, common.NewError("database_error", fmt.Sprintf("Got db error while getting refs for %v", paths)) + } + + for _, ref := range refs { + switch ref.Path { + case newPath: + return nil, common.NewError("invalid_parameters", "Invalid destination path. Object Already exists.") + default: + if ref.Type == reference.FILE { + return nil, common.NewError("invalid_path", fmt.Sprintf("%v is of file type", ref.Path)) + } + } + } + + allocationChange := &allocation.AllocationChange{} + allocationChange.ConnectionID = connectionObj.ID + allocationChange.Size = 0 + allocationChange.LookupHash = pathHash + allocationChange.Operation = constants.FileOperationMove + dfc := &allocation.MoveFileChange{ + ConnectionID: connectionObj.ID, + AllocationID: connectionObj.AllocationID, + SrcPath: objectRef.Path, + DestPath: destPath, + Type: objectRef.Type, + } + dfc.SrcPath = objectRef.Path + connectionObj.AddChange(allocationChange, dfc) + + err = connectionObj.Save(ctx) + if err != nil { + Logger.Error("Error in writing the connection meta data", zap.Error(err)) + return nil, common.NewError("connection_write_error", "Error writing the connection meta data") + } + + result := &allocation.UploadResult{} + result.Filename = objectRef.Name + result.Size = objectRef.Size + return result, nil } func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, connectionObj *allocation.AllocationChangeCollector) (*allocation.UploadResult, error) { From 36fa6357603509e8136f6f12f784d8b06438b7c5 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 14:29:19 +0530 Subject: [PATCH 51/98] add logs --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 5 +++++ code/go/0chain.net/blobbercore/reference/referencepath.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 1f80bc6c1..95eb0bb89 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -21,6 +21,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" ) @@ -468,6 +469,9 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* dirref = parent } else { + if fileref == nil { + fileref = &reference.Ref{Type: reference.DIRECTORY, Path: path, AllocationID: allocationID} + } r, err := reference.GetRefWithChildren(ctx, fileref, allocationID, filePath, offset, pageLimit) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid path. "+err.Error()) @@ -830,6 +834,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb pathRef, err = reference.GetPaginatedRefByLookupHash(ctx, pathHash) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { + logging.Logger.Error("GetRefs: GetPaginatedRefByLookupHash", zap.Error(err), zap.String("path", path), zap.String("pathHash", pathHash)) return nil, common.NewError("invalid_path", "") } return nil, err diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 83a5adbc2..5e03244c4 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -239,6 +239,7 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) // To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs // very easily and effectively; Same case for offsetDate. func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { + logging.Logger.Info("GetRefs", zap.String("allocation_id", allocationID), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("level", level), zap.Int("pageLimit", pageLimit)) levelQuery := " order by path LIMIT " + fmt.Sprintf("%d", pageLimit) if level == 0 { level = math.MaxInt @@ -335,6 +336,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, if len(pRefs) > 0 { newOffsetPath = pRefs[len(pRefs)-1].Path } + logging.Logger.Info("GetRefsResult", zap.Int("refs_count", len(pRefs))) return } From 84f875af759567d96f20aa702c8cdf72986644aa Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 15:16:43 +0530 Subject: [PATCH 52/98] set name --- code/go/0chain.net/blobbercore/reference/referencepath.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 5e03244c4..95c99b638 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -269,6 +269,11 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, } if (offsetPath == "" || path == offsetPath) && returnParentDirectory && level == math.MaxInt { pRefs = append(pRefs, *parentRef) + if pageLimit == 1 { + refs = &pRefs + newOffsetPath = parentRef.Path + return + } } tx := datastore.GetStore().GetTransaction(ctx) rows, err := tx.Raw(`WITH RECURSIVE hierarchy_cte AS ( @@ -327,6 +332,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, return } ref.AllocationID = allocationID + ref.Name = filepath.Base(ref.Path) pRefs = append(pRefs, ref) } if err != nil { From c92a62aa70c181030c1e892520d12305dda69f57 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 15:51:41 +0530 Subject: [PATCH 53/98] add deleted at --- code/go/0chain.net/blobbercore/reference/referencepath.go | 1 + 1 file changed, 1 insertion(+) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 95c99b638..3d06319d0 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -294,6 +294,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, reference_objects as hierarchy WHERE parent_id = ? + AND deleted_at IS NULL UNION From 4bcd3a48fe6f471594db6bd45aa1b579af700794 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 19:59:44 +0530 Subject: [PATCH 54/98] add collector cache --- .../allocation/allocationchange.go | 13 +++++-------- .../blobbercore/allocation/copyfilechange.go | 2 +- .../allocation/file_changer_upload.go | 19 +++++++++++-------- .../blobbercore/allocation/movefilechange.go | 2 +- .../blobbercore/allocation/newdirchange.go | 2 +- .../blobbercore/reference/dbCollector.go | 11 +++++++++++ .../0chain.net/blobbercore/reference/ref.go | 17 +++++++++++++---- .../1698861371_full_db_snapshot.sql | 2 +- goose/migrations/1718391849_ref_index.sql | 2 +- 9 files changed, 45 insertions(+), 25 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 47014087c..d9d3bc0a3 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -18,7 +18,6 @@ import ( "go.uber.org/zap" "gorm.io/gorm" - "gorm.io/gorm/clause" ) const ( @@ -295,9 +294,7 @@ func (a *AllocationChangeCollector) DeleteChanges(ctx context.Context) { } type Result struct { - Id string - LookupHash string - FilestoreVersion int + LookupHash string } // TODO: Need to speed up this function @@ -314,7 +311,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { tx := datastore.GetStore().GetTransaction(ctx) - err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "filestore_version", "lookup_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). + err := tx.Model(&reference.Ref{}).Select("lookup_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error { for _, ref := range refs { @@ -328,7 +325,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { wg.Done() }() - err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.LookupHash, ref.FilestoreVersion) + err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.LookupHash, filestore.VERSION) if err != nil { logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error())) } @@ -359,7 +356,7 @@ func deleteFromFileStore(ctx context.Context, allocationID string) error { return datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) - err := db.Model(&reference.Ref{}).Unscoped().Select("id", "lookup_hash", "filestore_version"). + err := db.Model(&reference.Ref{}).Unscoped().Select("lookup_hash"). Where("allocation_id=? AND is_precommit=? AND type=? AND deleted_at is not NULL", allocationID, true, reference.FILE). FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error { @@ -374,7 +371,7 @@ func deleteFromFileStore(ctx context.Context, allocationID string) error { }() err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.LookupHash, - res.FilestoreVersion) + filestore.VERSION) if err != nil { logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), zap.String("validation_root", res.LookupHash)) diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 4389d19d9..7ffaa953a 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -53,7 +53,7 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, } } - parentDir, err := reference.Mkdir(ctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts) + parentDir, err := reference.Mkdir(ctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts, collector) if err != nil { return err } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 115a5875d..100355d69 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -8,7 +8,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" - "go.uber.org/zap" "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -16,7 +15,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) // swagger:model UploadFileChanger @@ -70,10 +68,16 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, ID int64 Type string } - db := datastore.GetStore().GetTransaction(ctx) - err := db.Model(&reference.Ref{}).Select("id", "type").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err + cachedRef := collector.GetFromCache(newFile.LookupHash) + if cachedRef != nil { + refResult.ID = cachedRef.ID + refResult.Type = cachedRef.Type + } else { + db := datastore.GetStore().GetTransaction(ctx) + err := db.Model(&reference.Ref{}).Select("id", "type").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error + if err != nil && err != gorm.ErrRecordNotFound { + return err + } } if refResult.ID > 0 { if !nf.CanUpdate { @@ -87,9 +91,8 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, refResult.ID = 0 // get parent id parent := filepath.Dir(nf.Path) - logging.Logger.Info("applyChange", zap.String("parent", parent), zap.String("path", nf.Path)) // create or get parent directory - parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent, ts) + parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent, ts, collector) if err != nil { return err } diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange.go b/code/go/0chain.net/blobbercore/allocation/movefilechange.go index b926b1a11..21497c1d2 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange.go @@ -53,7 +53,7 @@ func (rf *MoveFileChange) ApplyChange(cctx context.Context, } rf.Type = srcRef.Type - parentDir, err := reference.Mkdir(cctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts) + parentDir, err := reference.Mkdir(cctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts, collector) if err != nil { return err } diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 69d31616f..f559e479c 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -31,7 +31,7 @@ func (nf *NewDir) ApplyChange(ctx context.Context, return err } if parentRef == nil || parentRef.ID == 0 { - _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, ts) + _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, ts, collector) } else { parentIDRef := &parentRef.ID newRef := reference.NewDirectoryRef() diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index b3905bd80..ea80a6575 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -10,11 +10,14 @@ type QueryCollector interface { CreateRefRecord(ref *Ref) DeleteRefRecord(ref *Ref) Finalize(ctx context.Context) error + AddToCache(ref *Ref) + GetFromCache(lookupHash string) *Ref } type dbCollector struct { createdRefs []*Ref deletedRefs []*Ref + refMap map[string]*Ref } func NewCollector(changes int) QueryCollector { @@ -49,3 +52,11 @@ func (dc *dbCollector) Finalize(ctx context.Context) error { return nil } + +func (dc *dbCollector) AddToCache(ref *Ref) { + dc.refMap[ref.LookupHash] = ref +} + +func (dc *dbCollector) GetFromCache(lookupHash string) *Ref { + return dc.refMap[lookupHash] +} diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 081d020e1..e8046dee6 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -154,16 +154,25 @@ func NewFileRef() *Ref { } // Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child -func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timestamp) (*Ref, error) { +func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timestamp, collector QueryCollector) (*Ref, error) { db := datastore.GetStore().GetTransaction(ctx) if destpath != "/" { destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") } destLookupHash := GetReferenceLookup(allocationID, destpath) var destRef Ref - err := db.Select("id", "type").Where(&Ref{LookupHash: destLookupHash}).Take(&destRef).Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, err + cachedRef := collector.GetFromCache(destLookupHash) + if cachedRef != nil { + destRef = *cachedRef + } else { + err := db.Select("id", "type").Where(&Ref{LookupHash: destLookupHash}).Take(&destRef).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + destRef.LookupHash = destLookupHash + if destRef.ID > 0 { + defer collector.AddToCache(&destRef) + } } if destRef.ID > 0 { if destRef.Type != DIRECTORY { diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 21e0dfa55..e9e2ac742 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -756,7 +756,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash); +CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) where deleted_at IS NULL INCLUDE(id,type,path); -- diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index 09d6570e6..4c10de918 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -2,5 +2,5 @@ -- +goose StatementBegin DROP INDEX idx_created_at,idx_updated_at,idx_lookup_hash,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; -CREATE INDEX idx_allocation_precommit_deleted ON reference_objects (allocation_id, is_precommit) where deleted_at is NULL; +CREATE INDEX idx_allocation_precommit_deleted ON reference_objects (allocation_id, is_precommit) where deleted_at is NULL INCLUDE(type,lookup_hash); -- +goose StatementEnd \ No newline at end of file From a604d5d5247003e015afe7a965145ae8b95a3da9 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 21:05:39 +0530 Subject: [PATCH 55/98] fix index --- .../0chain.net/blobbercore/allocation/file_changer_upload.go | 5 +---- goose/migrations/1698861371_full_db_snapshot.sql | 2 +- goose/migrations/1718391849_ref_index.sql | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 100355d69..5b6975efa 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -88,7 +88,6 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, } collector.DeleteRefRecord(deleteRecord) } - refResult.ID = 0 // get parent id parent := filepath.Dir(nf.Path) // create or get parent directory @@ -96,9 +95,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, if err != nil { return err } - refResult.ID = parentRef.ID - - newFile.ParentID = &refResult.ID + newFile.ParentID = &parentRef.ID collector.CreateRefRecord(newFile) return err diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index e9e2ac742..7ee3270b4 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -756,7 +756,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) where deleted_at IS NULL INCLUDE(id,type,path); +CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type,path) where deleted_at IS NULL; -- diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index 4c10de918..a53b10d34 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -2,5 +2,5 @@ -- +goose StatementBegin DROP INDEX idx_created_at,idx_updated_at,idx_lookup_hash,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; -CREATE INDEX idx_allocation_precommit_deleted ON reference_objects (allocation_id, is_precommit) where deleted_at is NULL INCLUDE(type,lookup_hash); +CREATE INDEX idx_allocation_precommit_deleted ON reference_objects (allocation_id, is_precommit) INCLUDE(type,lookup_hash) where deleted_at is NULL; -- +goose StatementEnd \ No newline at end of file From 0606901b49775e23f993909182c2b7a5f3777812 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 22:18:43 +0530 Subject: [PATCH 56/98] init map --- code/go/0chain.net/blobbercore/reference/dbCollector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index ea80a6575..8e5f59441 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -24,6 +24,7 @@ func NewCollector(changes int) QueryCollector { return &dbCollector{ createdRefs: make([]*Ref, 0, changes*4), deletedRefs: make([]*Ref, 0, changes*4), + refMap: make(map[string]*Ref), } } From adb7facb9b7d47c8649b5f71d10f00a093e26015 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sun, 28 Jul 2024 00:44:35 +0530 Subject: [PATCH 57/98] add new indexes --- goose/migrations/1698861371_full_db_snapshot.sql | 9 +-------- goose/migrations/1718391849_ref_index.sql | 6 ++++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 7ee3270b4..1482d93ea 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -572,7 +572,7 @@ ALTER TABLE ONLY marketplace_share_info ALTER COLUMN id SET DEFAULT nextval('mar ALTER TABLE ONLY reference_objects ALTER COLUMN id SET DEFAULT nextval('reference_objects_id_seq'::regclass); -ALTER TABLE ONLY reference_objects ADD CONSTRAINT path_commit UNIQUE(lookup_hash,is_precommit); +-- ALTER TABLE ONLY reference_objects ADD CONSTRAINT path_commit UNIQUE(lookup_hash,is_precommit); -- -- Name: terms id; Type: DEFAULT; Schema: public; Owner: blobber_user @@ -842,13 +842,6 @@ CREATE INDEX idx_updated_at ON reference_objects USING btree (updated_at DESC); CREATE INDEX idx_write_pools_cab ON write_pools USING btree (allocation_id); --- --- Name: path_idx; Type: INDEX; Schema: public; Owner: blobber_user --- - -CREATE INDEX path_idx ON reference_objects USING btree (path); - - -- -- Name: allocation_changes fk_allocation_connections_changes; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user -- diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index a53b10d34..bb0b28ed6 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -1,6 +1,8 @@ -- +goose Up -- +goose StatementBegin -DROP INDEX idx_created_at,idx_updated_at,idx_lookup_hash,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; +DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; -CREATE INDEX idx_allocation_precommit_deleted ON reference_objects (allocation_id, is_precommit) INCLUDE(type,lookup_hash) where deleted_at is NULL; +CREATE INDEX idx_is_precommit on reference_objects(allocation_id) INCLUDE(type,lookup_hash,id) WHERE is_precommit=true AND type='f' AND deleted_at IS NULL; + +CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL; -- +goose StatementEnd \ No newline at end of file From 60cf8c89829592cb423480e6515f855e6394dbca Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sun, 28 Jul 2024 03:01:21 +0530 Subject: [PATCH 58/98] update is_precommit index --- goose/migrations/1718391849_ref_index.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index bb0b28ed6..20cc6ea50 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -2,7 +2,7 @@ -- +goose StatementBegin DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; -CREATE INDEX idx_is_precommit on reference_objects(allocation_id) INCLUDE(type,lookup_hash,id) WHERE is_precommit=true AND type='f' AND deleted_at IS NULL; +CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(type,lookup_hash,id,type) WHERE is_precommit=true AND deleted_at IS NULL; CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL; -- +goose StatementEnd \ No newline at end of file From 8b917325f461e9945a75635307ac142698db0a6d Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 29 Jul 2024 11:24:14 +0530 Subject: [PATCH 59/98] use materialized path --- .../blobbercore/reference/referencepath.go | 122 ++++-------------- .../1698861371_full_db_snapshot.sql | 4 +- 2 files changed, 29 insertions(+), 97 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 3d06319d0..ccb7929bf 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -2,7 +2,6 @@ package reference import ( "context" - "fmt" "math" "path/filepath" "strings" @@ -12,6 +11,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" + "gorm.io/gorm" ) type ReferencePath struct { @@ -239,111 +239,43 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) // To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs // very easily and effectively; Same case for offsetDate. func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { - logging.Logger.Info("GetRefs", zap.String("allocation_id", allocationID), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("level", level), zap.Int("pageLimit", pageLimit)) - levelQuery := " order by path LIMIT " + fmt.Sprintf("%d", pageLimit) - if level == 0 { - level = math.MaxInt - } else { - levelQuery = fmt.Sprintf(" AND level = %d order by path LIMIT %d", level, pageLimit) - } - refTypes := []string{} - returnParentDirectory := false - if _type != "" { - refTypes = append(refTypes, _type) - if _type == DIRECTORY { - returnParentDirectory = true + var ( + pRefs = make([]PaginatedRef, 0, pageLimit/4) + dbError error + dbQuery *gorm.DB + ) + path = filepath.Clean(path) + tx := datastore.GetStore().GetTransaction(ctx) + pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + 1 + if pathLevel == level { + dbQuery = tx.Model(&Ref{}).Where("parent_id = ?", parentRef.ID) + if _type != "" { + dbQuery = dbQuery.Where("type = ?", _type) } + dbQuery = dbQuery.Where("path > ?", offsetPath) + dbQuery = dbQuery.Order("path") } else { - refTypes = append(refTypes, "f", "d") - returnParentDirectory = true - } - pRefs := make([]PaginatedRef, 0, pageLimit/4) - if parentRef.Type != DIRECTORY { - pRefs = append(pRefs, *parentRef) - refs = &pRefs - return - } else if level == parentRef.PathLevel && returnParentDirectory { - pRefs = append(pRefs, *parentRef) - refs = &pRefs - return - } - if (offsetPath == "" || path == offsetPath) && returnParentDirectory && level == math.MaxInt { - pRefs = append(pRefs, *parentRef) - if pageLimit == 1 { - refs = &pRefs - newOffsetPath = parentRef.Path - return + dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"%") + if _type != "" { + dbQuery = dbQuery.Where("type = ?", _type) } - } - tx := datastore.GetStore().GetTransaction(ctx) - rows, err := tx.Raw(`WITH RECURSIVE hierarchy_cte AS ( - SELECT - id, - parent_id, - path, - level, - hierarchy.type, - actual_file_size, - mimetype, - encrypted_key, - encrypted_key_point, - actual_file_hash, - custom_meta, - updated_at - FROM - reference_objects as hierarchy - WHERE - parent_id = ? - AND deleted_at IS NULL - - UNION - - SELECT - h.id, - h.parent_id, - h.path, - h.level, - h.type, - h.actual_file_size, - h.mimetype, - h.encrypted_key, - h.encrypted_key_point, - h.actual_file_hash, - h.custom_meta, - h.updated_at - FROM - reference_objects h - INNER JOIN - hierarchy_cte hc ON h.parent_id = hc.id - WHERE - hc.type='d' - AND h.level <= ? - AND h.deleted_at IS NULL - ) - SELECT * from hierarchy_cte where type IN ? AND path > ?`+levelQuery, parentRef.ID, level, refTypes, offsetPath).Rows() - if err != nil { - return - } - defer rows.Close() - - for rows.Next() { - var ref PaginatedRef - err = tx.ScanRows(rows, &ref) - if err != nil { - return + if level != 0 { + dbQuery = dbQuery.Where("level = ?", level) } - ref.AllocationID = allocationID - ref.Name = filepath.Base(ref.Path) - pRefs = append(pRefs, ref) + + dbQuery = dbQuery.Where("path > ?", offsetPath) + + dbQuery = dbQuery.Order("path") } - if err != nil { + dbError = dbQuery.Limit(pageLimit).Find(&pRefs).Error + if dbError != nil && dbError != gorm.ErrRecordNotFound { + err = dbError return } refs = &pRefs if len(pRefs) > 0 { newOffsetPath = pRefs[len(pRefs)-1].Path } - logging.Logger.Info("GetRefsResult", zap.Int("refs_count", len(pRefs))) return } diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 1482d93ea..f78416d77 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -784,7 +784,7 @@ CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english': -- Name: idx_parent_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path); +CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL; -- -- Name: idx_parent_id_alloc; Type: INDEX; Schema: public; Owner: blobber_user @@ -797,7 +797,7 @@ CREATE INDEX idx_parent_id_alloc ON reference_objects USING btree (parent_id) wh -- Name: idx_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path); +CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path) include(id,level,type) WHERE deleted_at IS NULL; -- From 67bb51bd3a667f72367fe6d512a529d4252eada4 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 29 Jul 2024 13:00:35 +0530 Subject: [PATCH 60/98] hardcode limit values --- .../0chain.net/blobbercore/config/config.go | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 3cef13b2d..0a7bb44c2 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -290,14 +290,20 @@ func ReadConfig(deploymentMode int) { } else if Configuration.MinConfirmation > 100 { Configuration.MinConfirmation = 100 } - - Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily") + // TODO: Read values from config only, this is for testing + // Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily") + Configuration.BlockLimitDaily = 312500000 Configuration.BlockLimitRequest = viper.GetInt64("rate_limiters.block_limit_request") - Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly") - Configuration.UploadLimitMonthly = viper.GetInt64("rate_limiters.upload_limit_monthly") - Configuration.CommitLimitMonthly = viper.GetInt64("rate_limiters.commit_limit_monthly") - Configuration.CommitLimitDaily = viper.GetInt64("rate_limiters.commit_limit_daily") - Configuration.CommitZeroLimitDaily = viper.GetInt64("rate_limiters.commit_zero_limit_daily") + // Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly") + Configuration.BlockLimitMonthly = 312500000 + // Configuration.UploadLimitMonthly = viper.GetInt64("rate_limiters.upload_limit_monthly") + Configuration.UploadLimitMonthly = 312500000 + // Configuration.CommitLimitMonthly = viper.GetInt64("rate_limiters.commit_limit_monthly") + Configuration.CommitLimitMonthly = 300000000 + // Configuration.CommitLimitDaily = viper.GetInt64("rate_limiters.commit_limit_daily") + Configuration.CommitLimitDaily = 300000000 + // Configuration.CommitZeroLimitDaily = viper.GetInt64("rate_limiters.commit_zero_limit_daily") + Configuration.CommitZeroLimitDaily = 300000000 } // StorageSCConfiguration will include all the required sc configs to operate blobber From 26733c6029d93af86f3c682603ba4ee4e0143845 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 29 Jul 2024 18:48:57 +0530 Subject: [PATCH 61/98] remove parent path index --- goose/migrations/1698861371_full_db_snapshot.sql | 4 ++-- goose/migrations/1718391849_ref_index.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index f78416d77..6d4cf2da3 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -756,7 +756,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type,path) where deleted_at IS NULL; +CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type) where deleted_at IS NULL; -- @@ -784,7 +784,7 @@ CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english': -- Name: idx_parent_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL; +-- CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL; -- -- Name: idx_parent_id_alloc; Type: INDEX; Schema: public; Owner: blobber_user diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index 20cc6ea50..d9978ea89 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -2,7 +2,7 @@ -- +goose StatementBegin DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; -CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(type,lookup_hash,id,type) WHERE is_precommit=true AND deleted_at IS NULL; +CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(type,lookup_hash,id) WHERE is_precommit=true AND deleted_at IS NULL; CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL; -- +goose StatementEnd \ No newline at end of file From 5050629be834f123c527723aaf50fb427409a6ae Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 29 Jul 2024 20:36:59 +0530 Subject: [PATCH 62/98] fix type --- goose/migrations/1698861371_full_db_snapshot.sql | 2 +- goose/migrations/1718391849_ref_index.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 6d4cf2da3..1deb2f005 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -756,7 +756,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type) where deleted_at IS NULL; +CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,reference_objects.type) where deleted_at IS NULL; -- diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index d9978ea89..93489da37 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -2,7 +2,7 @@ -- +goose StatementBegin DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; -CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(type,lookup_hash,id) WHERE is_precommit=true AND deleted_at IS NULL; +CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(lookup_hash,id,reference_objects.type) WHERE is_precommit=true AND deleted_at IS NULL; CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL; -- +goose StatementEnd \ No newline at end of file From 66d466fccb0ca27fe6432cfe6c893ad0301a5413 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 29 Jul 2024 21:02:41 +0530 Subject: [PATCH 63/98] remove ref type --- goose/migrations/1698861371_full_db_snapshot.sql | 2 +- goose/migrations/1718391849_ref_index.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 1deb2f005..6d4cf2da3 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -756,7 +756,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,reference_objects.type) where deleted_at IS NULL; +CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type) where deleted_at IS NULL; -- diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index 93489da37..c6c3476db 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -2,7 +2,7 @@ -- +goose StatementBegin DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; -CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(lookup_hash,id,reference_objects.type) WHERE is_precommit=true AND deleted_at IS NULL; +CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(lookup_hash,id,type) WHERE is_precommit=true AND deleted_at IS NULL; CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL; -- +goose StatementEnd \ No newline at end of file From 071b9c57751458d99787b70694b6e073c5b191fa Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 29 Jul 2024 22:04:22 +0530 Subject: [PATCH 64/98] revert index changes --- .../0chain.net/blobbercore/allocation/allocationchange.go | 8 ++++---- goose/migrations/1698861371_full_db_snapshot.sql | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index d9d3bc0a3..710fbaa50 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -306,13 +306,13 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { return err } var refs []*Result - limitCh := make(chan struct{}, 10) + limitCh := make(chan struct{}, 25) wg := &sync.WaitGroup{} err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { tx := datastore.GetStore().GetTransaction(ctx) err := tx.Model(&reference.Ref{}).Select("lookup_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). - FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error { + FindInBatches(&refs, 100, func(tx *gorm.DB, batch int) error { for _, ref := range refs { @@ -339,7 +339,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { wg.Wait() if err != nil { - logging.Logger.Error("Error while moving to filestore", zap.Error(err)) + logging.Logger.Error("Error while moving files to filestore", zap.Error(err)) return err } @@ -349,7 +349,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { } func deleteFromFileStore(ctx context.Context, allocationID string) error { - limitCh := make(chan struct{}, 10) + limitCh := make(chan struct{}, 25) wg := &sync.WaitGroup{} var results []Result diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index f78416d77..6d4cf2da3 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -756,7 +756,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type,path) where deleted_at IS NULL; +CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type) where deleted_at IS NULL; -- @@ -784,7 +784,7 @@ CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english': -- Name: idx_parent_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL; +-- CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL; -- -- Name: idx_parent_id_alloc; Type: INDEX; Schema: public; Owner: blobber_user From 5330ba832218e973d86e77e04ca9c6909ba09782 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Fri, 9 Aug 2024 20:55:10 +0530 Subject: [PATCH 65/98] Use alloc version in place of precommit (#1465) * use alloc version in place of precommit * add timing log for finalize * add upload changer timing logs * use multiple txns * add txn in mkdir * remove lock in upload changer * fix dest ref in mkdir * add gorm logger * go mod tidy * add logger config for gorm * use ptr for create * change constraint name * fix lookup hash constraint * remove index from drop * add created ref to cache * use hash for lookup * add cache logs * fix change index * remove deleted at clause * add check for file ref * add option for empty commit * fix create marker in rollback handler * add thumbnail suffix in write file * add fix for move and copy * add log * use refLookupHash * set num blocks * add fix for destpath * add max connection changes in config * rmv get wm in file stats * increment num_of_updates --- .../allocation/allocationchange.go | 177 ++++++++++-------- .../blobbercore/allocation/common_test.go | 7 +- .../blobbercore/allocation/copyfilechange.go | 56 ++++-- .../allocation/deletefilechange.go | 6 +- .../allocation/file_changer_base.go | 6 + .../allocation/file_changer_update.go | 1 - .../allocation/file_changer_upload.go | 57 ++++-- .../allocation/file_changer_upload_main.go | 4 +- .../blobbercore/allocation/movefilechange.go | 56 +++--- .../blobbercore/allocation/newdirchange.go | 9 +- .../allocation/renamefilechange.go | 15 +- .../allocation/renamefilechange_main.go | 4 +- .../blobbercore/allocation/rollback.go | 10 +- .../0chain.net/blobbercore/config/config.go | 22 +-- .../blobbercore/datastore/mocket.go | 4 +- .../blobbercore/datastore/postgres.go | 9 +- .../blobbercore/datastore/sqlmock.go | 2 +- .../0chain.net/blobbercore/datastore/store.go | 2 +- .../blobbercore/filestore/storage.go | 14 +- .../0chain.net/blobbercore/filestore/store.go | 9 +- .../blobbercore/filestore/tree_validation.go | 18 +- .../handler/file_command_delete.go | 5 + .../handler/file_command_update.go | 4 +- .../handler/file_command_upload.go | 10 +- .../handler/object_operation_handler.go | 108 +++++++---- .../blobbercore/handler/storage_handler.go | 14 +- .../blobbercore/reference/dbCollector.go | 78 +++++++- .../blobbercore/reference/object.go | 13 +- .../0chain.net/blobbercore/reference/ref.go | 92 ++++++--- .../blobbercore/reference/referencepath.go | 18 +- code/go/0chain.net/core/encryption/hash.go | 15 +- config/0chain_blobber.yaml | 2 + go.mod | 12 +- go.sum | 45 +++-- .../1698861371_full_db_snapshot.sql | 22 +-- .../1717416291_change_lookuphash.sql | 2 +- goose/migrations/1718188301_change_idx.sql | 7 +- goose/migrations/1718391849_ref_index.sql | 6 +- 38 files changed, 611 insertions(+), 330 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 710fbaa50..5de70ec6b 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -32,7 +32,7 @@ type AllocationChangeProcessor interface { CommitToFileStore(ctx context.Context, mut *sync.Mutex) error DeleteTempFile() error ApplyChange(ctx context.Context, - ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error GetPath() []string Marshal() (string, error) Unmarshal(string) error @@ -153,7 +153,6 @@ func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clien ).Preload("Changes").Take(cc).Error if err == nil { - logging.Logger.Info("getAllocationChanges", zap.String("connection_id", connectionID), zap.Int("changes", len(cc.Changes))) cc.ComputeProperties() // Load connection Obj size from memory cc.Size = GetConnectionObjSize(connectionID) @@ -256,17 +255,35 @@ func (cc *AllocationChangeCollector) ComputeProperties() { } func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, - ts common.Timestamp, fileIDMeta map[string]string) error { + ts common.Timestamp, allocationVersion int64) error { + now := time.Now() collector := reference.NewCollector(len(cc.Changes)) + timeoutctx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + eg, egCtx := errgroup.WithContext(timeoutctx) + eg.SetLimit(10) for idx, change := range cc.Changes { - change.AllocationID = cc.AllocationID - changeProcessor := cc.AllocationChanges[idx] - err := changeProcessor.ApplyChange(ctx, ts, fileIDMeta, collector) - if err != nil { - return err + select { + case <-egCtx.Done(): + return egCtx.Err() + default: + changeIndex := idx + eg.Go(func() error { + change.AllocationID = cc.AllocationID + changeProcessor := cc.AllocationChanges[changeIndex] + return changeProcessor.ApplyChange(ctx, ts, allocationVersion, collector) + }) } } - return collector.Finalize(ctx) + err := eg.Wait() + if err != nil { + return err + } + elapsedApplyChanges := time.Since(now) + err = collector.Finalize(ctx, cc.AllocationID, allocationVersion) + elapsedFinalize := time.Since(now) - elapsedApplyChanges + logging.Logger.Info("ApplyChanges", zap.String("allocation_id", cc.AllocationID), zap.Duration("apply_changes", elapsedApplyChanges), zap.Duration("finalize", elapsedFinalize), zap.Int("changes", len(cc.Changes))) + return err } func (a *AllocationChangeCollector) CommitToFileStore(ctx context.Context) error { @@ -298,95 +315,103 @@ type Result struct { } // TODO: Need to speed up this function -func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { +func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context, allocationVersion int64) error { logging.Logger.Info("Move to filestore", zap.String("allocation_id", a.AllocationID)) - err := deleteFromFileStore(ctx, a.AllocationID) + var ( + refs []*reference.Ref + useRefCache bool + deletedRefs []*reference.Ref + ) + refCache := reference.GetRefCache(a.AllocationID) + defer reference.DeleteRefCache(a.AllocationID) + if refCache != nil && refCache.AllocationVersion == allocationVersion { + useRefCache = true + refs = refCache.CreatedRefs + deletedRefs = refCache.DeletedRefs + } else if refCache != nil && refCache.AllocationVersion != allocationVersion { + logging.Logger.Error("Ref cache is not valid", zap.String("allocation_id", a.AllocationID), zap.String("ref_cache_version", fmt.Sprintf("%d", refCache.AllocationVersion)), zap.String("allocation_version", fmt.Sprintf("%d", allocationVersion))) + } else { + logging.Logger.Error("Ref cache is nil", zap.String("allocation_id", a.AllocationID)) + } + err := deleteFromFileStore(a.AllocationID, deletedRefs, useRefCache) if err != nil { return err } - var refs []*Result - limitCh := make(chan struct{}, 25) - wg := &sync.WaitGroup{} - err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + limitCh := make(chan struct{}, 12) + wg := &sync.WaitGroup{} + if !useRefCache { tx := datastore.GetStore().GetTransaction(ctx) - err := tx.Model(&reference.Ref{}).Select("lookup_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). - FindInBatches(&refs, 100, func(tx *gorm.DB, batch int) error { - - for _, ref := range refs { - - limitCh <- struct{}{} - wg.Add(1) - - go func(ref *Result) { - defer func() { - <-limitCh - wg.Done() - }() - - err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.LookupHash, filestore.VERSION) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error())) - } - - }(ref) - } - - return nil - }).Error - - wg.Wait() - + err = tx.Model(&reference.Ref{}).Select("lookup_hash").Where("allocation_id=? AND allocation_version=? AND type=?", a.AllocationID, allocationVersion, reference.FILE).Find(&refs).Error if err != nil { logging.Logger.Error("Error while moving files to filestore", zap.Error(err)) return err } + } - return tx.Exec("UPDATE reference_objects SET is_precommit=? WHERE allocation_id=? AND is_precommit=? AND deleted_at is NULL", false, a.AllocationID, true).Error - }) - return err + for _, ref := range refs { + + limitCh <- struct{}{} + wg.Add(1) + refLookupHash := ref.LookupHash + go func() { + defer func() { + <-limitCh + wg.Done() + }() + err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, refLookupHash, filestore.VERSION) + if err != nil { + logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error())) + } + + }() + } + + wg.Wait() + return nil } -func deleteFromFileStore(ctx context.Context, allocationID string) error { - limitCh := make(chan struct{}, 25) +func deleteFromFileStore(allocationID string, deletedRefs []*reference.Ref, useRefCache bool) error { + limitCh := make(chan struct{}, 12) wg := &sync.WaitGroup{} - var results []Result + var results []*reference.Ref + if useRefCache { + results = deletedRefs + } return datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) + if !useRefCache { + err := db.Model(&reference.Ref{}).Unscoped().Select("lookup_hash"). + Where("allocation_id=? AND type=? AND deleted_at is not NULL", allocationID, reference.FILE). + Find(&results).Error + if err != nil && err != gorm.ErrRecordNotFound { + logging.Logger.Error("DeleteFromFileStore", zap.Error(err)) + return err + } + } - err := db.Model(&reference.Ref{}).Unscoped().Select("lookup_hash"). - Where("allocation_id=? AND is_precommit=? AND type=? AND deleted_at is not NULL", allocationID, true, reference.FILE). - FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error { - - for _, res := range results { - limitCh <- struct{}{} - wg.Add(1) - - go func(res Result) { - defer func() { - <-limitCh - wg.Done() - }() - - err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.LookupHash, - filestore.VERSION) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), - zap.String("validation_root", res.LookupHash)) - } - }(res) - + for _, res := range results { + limitCh <- struct{}{} + wg.Add(1) + resLookupHash := res.LookupHash + go func() { + defer func() { + <-limitCh + wg.Done() + }() + + err := filestore.GetFileStore().DeleteFromFilestore(allocationID, resLookupHash, + filestore.VERSION) + if err != nil { + logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), + zap.String("validation_root", res.LookupHash)) } - return nil - }).Error + }() - wg.Wait() - if err != nil && err != gorm.ErrRecordNotFound { - logging.Logger.Error("DeleteFromFileStore", zap.Error(err)) - return err } + wg.Wait() return db.Model(&reference.Ref{}).Unscoped(). Delete(&reference.Ref{}, diff --git a/code/go/0chain.net/blobbercore/allocation/common_test.go b/code/go/0chain.net/blobbercore/allocation/common_test.go index b0b5c3a5a..9cb6429fb 100644 --- a/code/go/0chain.net/blobbercore/allocation/common_test.go +++ b/code/go/0chain.net/blobbercore/allocation/common_test.go @@ -21,10 +21,9 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string, b := bytes.NewBuffer(make([]byte, 0)) n, _ := io.Copy(b, infile) return &filestore.FileOutputData{ - Name: fileData.Name, - Path: fileData.Path, - FixedMerkleRoot: "", - Size: n, + Name: fileData.Name, + Path: fileData.Path, + Size: n, }, nil } diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 7ffaa953a..f119bf33a 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -2,11 +2,13 @@ package allocation import ( "context" + "database/sql" "encoding/json" "path/filepath" "strings" "sync" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -27,33 +29,47 @@ func (rf *CopyFileChange) DeleteTempFile() error { } func (rf *CopyFileChange) ApplyChange(ctx context.Context, - ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) - srcRef, err := reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash) - if err != nil { - return err - } - exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath) - if err != nil { - return err - } - if exist { - return common.NewError("invalid_reference_path", "file already exists") - } - rf.Type = srcRef.Type - if srcRef.Type == reference.DIRECTORY { - isEmpty, err := reference.IsDirectoryEmpty(ctx, srcRef.ID) + var ( + srcRef *reference.Ref + err error + ) + + err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + srcRef, err = reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash) if err != nil { return err } - if !isEmpty { - return common.NewError("invalid_reference_path", "directory is not empty") + exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath) + if err != nil { + return err + } + if exist { + return common.NewError("invalid_reference_path", "file already exists") + } + + rf.Type = srcRef.Type + if srcRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, srcRef.ID) + if err != nil { + return err + } + if !isEmpty { + return common.NewError("invalid_reference_path", "directory is not empty") + } } + return nil + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return err } - parentDir, err := reference.Mkdir(ctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts, collector) + parentDir, err := reference.Mkdir(ctx, rf.AllocationID, filepath.Dir(rf.DestPath), allocationVersion, ts, collector) if err != nil { return err } @@ -67,11 +83,11 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, srcRef.ParentPath = filepath.Dir(rf.DestPath) srcRef.Name = filepath.Base(rf.DestPath) srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/")) - srcRef.FileMetaHash = encryption.Hash(srcRef.GetFileHashData()) + srcRef.FileMetaHash = encryption.FastHash(srcRef.GetFileHashData()) if rf.CustomMeta != "" { srcRef.CustomMeta = rf.CustomMeta } - srcRef.IsPrecommit = true + srcRef.AllocationVersion = allocationVersion collector.CreateRefRecord(srcRef) return nil diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index d7a924e3f..18c4c7a31 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -24,8 +24,10 @@ type DeleteFileChange struct { } func (nf *DeleteFileChange) ApplyChange(ctx context.Context, - ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { - return reference.DeleteObject(ctx, nf.AllocationID, nf.LookupHash, nf.Type, ts) + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + collector.LockTransaction() + defer collector.UnlockTransaction() + return reference.DeleteObject(ctx, nf.AllocationID, nf.LookupHash, nf.Type, ts, allocationVersion, collector) } func (nf *DeleteFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index e39c1d671..f89725e14 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -8,6 +8,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) // swagger:model BaseFileChanger @@ -117,6 +119,9 @@ func (fc *BaseFileChanger) DeleteTempFile() error { func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { + if fc.LookupHash == "" { + fc.LookupHash = reference.GetReferenceLookup(fc.AllocationID, fc.Path) + } if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} fileInputData.Name = fc.ThumbnailFilename @@ -139,6 +144,7 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mute fileInputData.Size = fc.Size fileInputData.Hasher = GetHasher(fc.ConnectionID, fc.LookupHash) if fileInputData.Hasher == nil { + logging.Logger.Error("CommitToFileStore: Error getting hasher", zap.String("connection_id", fc.ConnectionID), zap.String("lookup_hash", fc.LookupHash)) return common.NewError("file_store_error", "Error getting hasher") } _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index be91ffc8e..b8faf2677 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -90,7 +90,6 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.EncryptedKey = nf.EncryptedKey fileRef.EncryptedKeyPoint = nf.EncryptedKeyPoint fileRef.ChunkSize = nf.ChunkSize - fileRef.IsPrecommit = true fileRef.FilestoreVersion = filestore.VERSION return rootRef, nil diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 5b6975efa..50e11b0f4 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -2,12 +2,16 @@ package allocation import ( "context" + "database/sql" "encoding/json" + "math" "path/filepath" "strings" + "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "go.uber.org/zap" "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -15,6 +19,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) // swagger:model UploadFileChanger @@ -25,12 +30,12 @@ type UploadFileChanger struct { // ApplyChange update references, and create a new FileRef func (nf *UploadFileChanger) applyChange(ctx context.Context, - ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { if nf.AllocationID == "" { return common.NewError("invalid_parameter", "allocation_id is required") } - + now := time.Now() parentPath := filepath.Dir(nf.Path) nf.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) newFile := &reference.Ref{ @@ -54,50 +59,62 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, ChunkSize: nf.ChunkSize, CreatedAt: ts, UpdatedAt: ts, - IsPrecommit: true, LookupHash: nf.LookupHash, DataHash: nf.DataHash, DataHashSignature: nf.DataHashSignature, PathLevel: len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")), + NumBlocks: int64(math.Ceil(float64(nf.Size*1.0) / float64(nf.ChunkSize))), FilestoreVersion: filestore.VERSION, + AllocationVersion: allocationVersion, + NumUpdates: 1, } - newFile.FileMetaHash = encryption.Hash(newFile.GetFileMetaHashData()) - + newFile.FileMetaHash = encryption.FastHash(newFile.GetFileMetaHashData()) + elapsedNewFile := time.Since(now) // find if ref exists var refResult struct { - ID int64 - Type string + ID int64 + Type string + NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"` } - cachedRef := collector.GetFromCache(newFile.LookupHash) - if cachedRef != nil { - refResult.ID = cachedRef.ID - refResult.Type = cachedRef.Type - } else { - db := datastore.GetStore().GetTransaction(ctx) - err := db.Model(&reference.Ref{}).Select("id", "type").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err - } + + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + tx := datastore.GetStore().GetTransaction(ctx) + return tx.Model(&reference.Ref{}).Select("id", "type", "num_of_updates").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil && err != gorm.ErrRecordNotFound { + return err } + if refResult.ID > 0 { if !nf.CanUpdate { return common.NewError("prohibited_allocation_file_options", "Cannot update data in this allocation.") } + if refResult.Type != reference.FILE { + return common.NewError("invalid_reference_path", "Directory already exists with the same path") + } deleteRecord := &reference.Ref{ - ID: refResult.ID, + ID: refResult.ID, + LookupHash: newFile.LookupHash, + Type: refResult.Type, } collector.DeleteRefRecord(deleteRecord) + newFile.NumUpdates = refResult.NumUpdates + 1 } + elapsedNewFileRecord := time.Since(now) - elapsedNewFile // get parent id parent := filepath.Dir(nf.Path) // create or get parent directory - parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent, ts, collector) + parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent, allocationVersion, ts, collector) if err != nil { return err } + elapsedMkdir := time.Since(now) - elapsedNewFileRecord - elapsedNewFile newFile.ParentID = &parentRef.ID collector.CreateRefRecord(newFile) - + elapsedCreateRefRecord := time.Since(now) - elapsedMkdir - elapsedNewFileRecord - elapsedNewFile + logging.Logger.Info("UploadFileChanger", zap.Duration("elapsedNewFile", elapsedNewFile), zap.Duration("elapsedNewFileRecord", elapsedNewFileRecord), zap.Duration("elapsedMkdir", elapsedMkdir), zap.Duration("elapsedCreateRefRecord", elapsedCreateRefRecord), zap.Duration("elapsedTotal", time.Since(now))) return err } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go index 324862949..7cb2fea21 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go @@ -11,6 +11,6 @@ import ( ) func (nf *UploadFileChanger) ApplyChange(ctx context.Context, - ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { - return nf.applyChange(ctx, ts, fileIDMeta, collector) + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + return nf.applyChange(ctx, ts, allocationVersion, collector) } diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange.go b/code/go/0chain.net/blobbercore/allocation/movefilechange.go index 21497c1d2..52d587029 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange.go @@ -2,11 +2,13 @@ package allocation import ( "context" + "database/sql" "encoding/json" "path/filepath" "strings" "sync" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -26,40 +28,49 @@ func (rf *MoveFileChange) DeleteTempFile() error { } func (rf *MoveFileChange) ApplyChange(cctx context.Context, - ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + var err error srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) - - srcRef, err := reference.GetReferenceByLookupHash(cctx, rf.AllocationID, srcLookUpHash) - if err != nil { - return err - } - exist, err := reference.IsRefExist(cctx, rf.AllocationID, rf.DestPath) - if err != nil { - return err - } - if exist { - return common.NewError("invalid_reference_path", "file already exists") - } - - if srcRef.Type == reference.DIRECTORY { - isEmpty, err := reference.IsDirectoryEmpty(cctx, srcRef.ID) + var srcRef *reference.Ref + err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + srcRef, err = reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash) if err != nil { return err } - if !isEmpty { - return common.NewError("invalid_reference_path", "directory is not empty") + exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath) + if err != nil { + return err + } + if exist { + return common.NewError("invalid_reference_path", "file already exists") + } + if srcRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(cctx, srcRef.ID) + if err != nil { + return err + } + if !isEmpty { + return common.NewError("invalid_reference_path", "directory is not empty") + } } + return nil + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return err } rf.Type = srcRef.Type - - parentDir, err := reference.Mkdir(cctx, rf.AllocationID, filepath.Dir(rf.DestPath), ts, collector) + parentDir, err := reference.Mkdir(cctx, rf.AllocationID, filepath.Dir(rf.DestPath), allocationVersion, ts, collector) if err != nil { return err } deleteRef := &reference.Ref{ - ID: srcRef.ID, + ID: srcRef.ID, + LookupHash: srcLookUpHash, + Type: srcRef.Type, } collector.DeleteRefRecord(deleteRef) @@ -72,7 +83,8 @@ func (rf *MoveFileChange) ApplyChange(cctx context.Context, srcRef.CreatedAt = ts srcRef.UpdatedAt = ts srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/")) - srcRef.FileMetaHash = encryption.Hash(srcRef.GetFileHashData()) + srcRef.FileMetaHash = encryption.FastHash(srcRef.GetFileHashData()) + srcRef.AllocationVersion = allocationVersion collector.CreateRefRecord(srcRef) return nil diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index f559e479c..6d5778fd0 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -23,15 +23,15 @@ type NewDir struct { } func (nf *NewDir) ApplyChange(ctx context.Context, - ts common.Timestamp, _ map[string]string, collector reference.QueryCollector) error { + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { parentPath := filepath.Dir(nf.Path) parentPathLookup := reference.GetReferenceLookup(nf.AllocationID, parentPath) - parentRef, err := reference.GetReferenceByLookupHash(ctx, nf.AllocationID, parentPathLookup) + parentRef, err := reference.GetFullReferenceByLookupHashWithNewTransaction(parentPathLookup) if err != nil && err != gorm.ErrRecordNotFound { return err } if parentRef == nil || parentRef.ID == 0 { - _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, ts, collector) + _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, allocationVersion, ts, collector) } else { parentIDRef := &parentRef.ID newRef := reference.NewDirectoryRef() @@ -46,13 +46,14 @@ func (nf *NewDir) ApplyChange(ctx context.Context, newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) newRef.CreatedAt = ts newRef.UpdatedAt = ts - newRef.FileMetaHash = encryption.Hash(newRef.GetFileMetaHashData()) + newRef.FileMetaHash = encryption.FastHash(newRef.GetFileMetaHashData()) if nf.CustomMeta != "" { newRef.CustomMeta = nf.CustomMeta } if nf.MimeType != "" { newRef.MimeType = nf.MimeType } + newRef.AllocationVersion = allocationVersion collector.CreateRefRecord(newRef) } return err diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index c197bd082..774027eee 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -32,8 +32,9 @@ func (rf *RenameFileChange) DeleteTempFile() error { } func (rf *RenameFileChange) applyChange(ctx context.Context, - ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { - + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + collector.LockTransaction() + defer collector.UnlockTransaction() if rf.Path == "/" { return common.NewError("invalid_operation", "cannot rename root path") } @@ -47,8 +48,8 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, if isFilePresent { return common.NewError("invalid_reference_path", "file already exists") } - - ref, err := reference.GetReference(ctx, rf.AllocationID, rf.Path) + oldFileLookupHash := reference.GetReferenceLookup(rf.AllocationID, rf.Path) + ref, err := reference.GetReferenceByLookupHash(ctx, rf.AllocationID, oldFileLookupHash) if err != nil { return common.NewError("invalid_reference_path", err.Error()) } @@ -63,7 +64,9 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, } rf.Type = ref.Type deleteRef := &reference.Ref{ - ID: ref.ID, + ID: ref.ID, + LookupHash: oldFileLookupHash, + Type: ref.Type, } collector.DeleteRefRecord(deleteRef) ref.Name = rf.NewName @@ -78,7 +81,7 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, if rf.MimeType != "" { ref.MimeType = rf.MimeType } - ref.IsPrecommit = true + ref.AllocationVersion = allocationVersion collector.CreateRefRecord(ref) rf.newLookupHash = ref.LookupHash return nil diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go index 0c91b5cb7..424ffe7ab 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go @@ -11,7 +11,7 @@ import ( ) func (rf *RenameFileChange) ApplyChange(ctx context.Context, - ts common.Timestamp, fileIDMeta map[string]string, collector reference.QueryCollector) error { + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { - return rf.applyChange(ctx, ts, fileIDMeta, collector) + return rf.applyChange(ctx, ts, allocationVersion, collector) } diff --git a/code/go/0chain.net/blobbercore/allocation/rollback.go b/code/go/0chain.net/blobbercore/allocation/rollback.go index fe25adaac..d3070ad96 100644 --- a/code/go/0chain.net/blobbercore/allocation/rollback.go +++ b/code/go/0chain.net/blobbercore/allocation/rollback.go @@ -8,23 +8,23 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" ) -func ApplyRollback(ctx context.Context, allocationID string) error { +func ApplyRollback(ctx context.Context, allocationID string, allocationVersion int64) error { db := datastore.GetStore().GetTransaction(ctx) - // delete all is_precommit rows + // delete all current allocation version rows err := db.Model(&reference.Ref{}).Unscoped(). Delete(&reference.Ref{}, - "allocation_id=? AND is_precommit=? AND deleted_at IS NULL", - allocationID, true).Error + "allocation_id=? AND allocation_version=? AND deleted_at IS NULL", + allocationID, allocationVersion).Error if err != nil { return err } // err = db.Exec("UPDATE file_stats SET deleted_at=NULL WHERE ref_id IN (SELECT id FROM reference_objects WHERE allocation_id=? AND deleted_at IS NOT NULL)", allocationID).Error // revive soft deleted ref rows - err = db.Exec("UPDATE reference_objects SET deleted_at=NULL,is_precommit=? WHERE allocation_id=? AND deleted_at IS NOT NULL", false, allocationID).Error + err = db.Exec("UPDATE reference_objects SET deleted_at=NULL WHERE allocation_id=? AND deleted_at IS NOT NULL", allocationID).Error return err } diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 0a7bb44c2..2af3f39d5 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -35,6 +35,7 @@ func SetupDefaultConfig() { viper.SetDefault("rate_limiters.commit_limit_monthly", 30000) viper.SetDefault("rate_limiters.commit_limit_daily", 1600) viper.SetDefault("rate_limiters.commit_zero_limit_daily", 400) + viper.SetDefault("rate_limiters.max_connection_changes", 100) viper.SetDefault("healthcheck.frequency", "60s") @@ -121,6 +122,7 @@ type Config struct { CommitLimitDaily int64 CommitZeroLimitDaily int64 ChallengeCleanupGap int64 + MaxConnectionChanges int HealthCheckWorkerFreq time.Duration @@ -290,20 +292,14 @@ func ReadConfig(deploymentMode int) { } else if Configuration.MinConfirmation > 100 { Configuration.MinConfirmation = 100 } - // TODO: Read values from config only, this is for testing - // Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily") - Configuration.BlockLimitDaily = 312500000 + Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily") Configuration.BlockLimitRequest = viper.GetInt64("rate_limiters.block_limit_request") - // Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly") - Configuration.BlockLimitMonthly = 312500000 - // Configuration.UploadLimitMonthly = viper.GetInt64("rate_limiters.upload_limit_monthly") - Configuration.UploadLimitMonthly = 312500000 - // Configuration.CommitLimitMonthly = viper.GetInt64("rate_limiters.commit_limit_monthly") - Configuration.CommitLimitMonthly = 300000000 - // Configuration.CommitLimitDaily = viper.GetInt64("rate_limiters.commit_limit_daily") - Configuration.CommitLimitDaily = 300000000 - // Configuration.CommitZeroLimitDaily = viper.GetInt64("rate_limiters.commit_zero_limit_daily") - Configuration.CommitZeroLimitDaily = 300000000 + Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly") + Configuration.UploadLimitMonthly = viper.GetInt64("rate_limiters.upload_limit_monthly") + Configuration.CommitLimitMonthly = viper.GetInt64("rate_limiters.commit_limit_monthly") + Configuration.CommitLimitDaily = viper.GetInt64("rate_limiters.commit_limit_daily") + Configuration.CommitZeroLimitDaily = viper.GetInt64("rate_limiters.commit_zero_limit_daily") + Configuration.MaxConnectionChanges = viper.GetInt("rate_limiters.max_connection_changes") } // StorageSCConfiguration will include all the required sc configs to operate blobber diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go index b560b0841..fb9b3d4cc 100644 --- a/code/go/0chain.net/blobbercore/datastore/mocket.go +++ b/code/go/0chain.net/blobbercore/datastore/mocket.go @@ -72,7 +72,7 @@ func (store *Mocket) Close() { } } -func (store *Mocket) CreateTransaction(ctx context.Context,opts ...*sql.TxOptions) context.Context { +func (store *Mocket) CreateTransaction(ctx context.Context, opts ...*sql.TxOptions) context.Context { db := store.db.Begin() return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(db)) } @@ -86,7 +86,7 @@ func (store *Mocket) GetTransaction(ctx context.Context) *EnhancedDB { return nil } -func (store *Mocket) WithNewTransaction(f func(ctx context.Context) error) error { +func (store *Mocket) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error { ctx := store.CreateTransaction(context.TODO()) defer ctx.Done() diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index 241701dfe..47e46623b 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -11,6 +11,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/logging" "gorm.io/driver/postgres" "gorm.io/gorm" + "moul.io/zapgorm2" ) // postgresStore store implementation for postgres @@ -46,6 +47,11 @@ func (p *postgresStore) GetPgDB() (*gorm.DB, error) { } func (store *postgresStore) Open() error { + gormLogger := zapgorm2.New(logging.Logger) + gormLogger.SlowThreshold = 100 * time.Millisecond + gormLogger.IgnoreRecordNotFoundError = true + gormLogger.SkipCallerLookup = true + gormLogger.SetAsDefault() db, err := gorm.Open(postgres.Open(fmt.Sprintf( "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", config.Configuration.DBHost, config.Configuration.DBPort, @@ -53,6 +59,7 @@ func (store *postgresStore) Open() error { config.Configuration.DBPassword)), &gorm.Config{ SkipDefaultTransaction: true, // https://gorm.io/docs/performance.html#Disable-Default-Transaction PrepareStmt: true, //https://gorm.io/docs/performance.html#Caches-Prepared-Statement + Logger: gormLogger, }) if err != nil { return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) @@ -99,7 +106,7 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *EnhancedDB { return nil } -func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error) error { +func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error { timeoutctx, cancel := context.WithTimeout(context.TODO(), 45*time.Second) defer cancel() ctx := store.CreateTransaction(timeoutctx) diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go index a7bcdeac5..354cc69e5 100644 --- a/code/go/0chain.net/blobbercore/datastore/sqlmock.go +++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go @@ -81,7 +81,7 @@ func (store *Sqlmock) GetTransaction(ctx context.Context) *EnhancedDB { return nil } -func (store *Sqlmock) WithNewTransaction(f func(ctx context.Context) error) error { +func (store *Sqlmock) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error { ctx := store.CreateTransaction(context.TODO()) defer ctx.Done() diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index a097be572..bedc36896 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -45,7 +45,7 @@ type Store interface { CreateTransaction(ctx context.Context, opts ...*sql.TxOptions) context.Context // GetTransaction get transaction from context GetTransaction(ctx context.Context) *EnhancedDB - WithNewTransaction(f func(ctx context.Context) error) error + WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error WithTransaction(ctx context.Context, f func(ctx context.Context) error) error // Get db connection with user that creates roles and databases. Its dialactor does not contain database name GetPgDB() (*gorm.DB, error) diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index e432d9c6d..c9cdf171a 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -61,7 +61,11 @@ const ( ) func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, infile multipart.File) (*FileOutputData, error) { - tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileData.FilePathHash, conID) + fileHash := fileData.LookupHash + if fileData.IsThumbnail { + fileHash = fileData.LookupHash + ThumbnailSuffix + } + tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileHash, conID) var ( initialSize int64 ) @@ -214,7 +218,7 @@ func (fs *FileStore) DeletePreCommitDir(allocID string) error { func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) (_ bool, err error) { now := time.Now() - logging.Logger.Info("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData)) + logging.Logger.Debug("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData)) fileHash := fileData.LookupHash if fileData.IsThumbnail { fileHash = fileData.LookupHash + ThumbnailSuffix @@ -316,7 +320,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) // 5. Move: It is Copy + Delete. Delete will not delete file if ref exists in database. i.e. copy would create // ref that refers to this file therefore it will be skipped fs.incrDecrAllocFileSizeAndNumber(allocID, fileSize, 1) - logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.Duration("elapsed_total", time.Since(now))) + logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.String("lookup_hash", fileData.LookupHash), zap.Duration("elapsed_total", time.Since(now))) return true, nil } @@ -341,6 +345,10 @@ func (fs *FileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash size := stat.Size() newObjectPath := fs.getPreCommitPathForFile(allocationID, newFileLookupHash, VERSION) + err = createDirs(filepath.Dir(newObjectPath)) + if err != nil { + return common.NewError("blob_object_precommit_dir_creation_error", err.Error()) + } newFile, err := os.Create(newObjectPath) if err != nil { return common.NewError("file_create_error", err.Error()) diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index 3cd80e210..3e39ae39d 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -22,11 +22,10 @@ type FileInputData struct { //Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. UploadOffset int64 //IsFinal the request is final chunk - IsFinal bool - IsThumbnail bool - FilePathHash string - Size int64 - Hasher *CommitHasher + IsFinal bool + IsThumbnail bool + Size int64 + Hasher *CommitHasher } type FileOutputData struct { diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 778e35d6d..1ceffce1e 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -415,9 +415,17 @@ type CommitHasher struct { dataSize int64 } +var ( + md5Pool = &sync.Pool{ + New: func() interface{} { + return md5.New() + }, + } +) + func NewCommitHasher(dataSize int64) *CommitHasher { c := new(CommitHasher) - c.md5hasher = md5.New() + c.md5hasher = md5Pool.Get().(hash.Hash) c.isInitialized = true c.doneChan = make(chan struct{}) c.dataSize = dataSize @@ -436,7 +444,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil defer f.Close() var toFinalize bool var totalWritten int64 - + logging.Logger.Info("hasher_start", zap.String("fileHash", filePathHash), zap.String("fileName", fileName), zap.String("tempFilePath", tempFilePath)) for { select { case <-ctx.Done(): @@ -458,7 +466,6 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil } else if pq.DataBytes == 0 { continue } - logging.Logger.Info("hasher_pop", zap.Int64("offset", pq.Offset), zap.Int64("dataBytes", pq.DataBytes), zap.Any("toFinalize", toFinalize), zap.Int64("dataSize", c.dataSize), zap.String("filename", fileName), zap.Int64("totalWritten", totalWritten)) bufSize := 2 * BufferSize if pq.DataBytes < int64(bufSize) { bufSize = int(pq.DataBytes) @@ -571,5 +578,8 @@ func (c *CommitHasher) GetValidationMerkleRoot() string { } func (c *CommitHasher) GetMd5Hash() string { - return hex.EncodeToString(c.md5hasher.Sum(nil)) + hash := hex.EncodeToString(c.md5hasher.Sum(nil)) + c.md5hasher.Reset() + md5Pool.Put(c.md5hasher) + return hash } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 3ee62b2eb..aef760aaf 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "path/filepath" "github.com/0chain/gosdk/constants" "gorm.io/gorm" @@ -41,6 +42,10 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_parameters", "Invalid path") } + if filepath.Clean(path) != path { + return common.NewError("invalid_parameters", "Invalid path") + } + cmd.path = path connectionID, ok := common.GetField(req, "connection_id") diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index 335b8ecfb..aaadff53d 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -119,7 +119,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj Path: cmd.fileChanger.Path, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, - FilePathHash: filePathHash, + LookupHash: filePathHash, Size: cmd.fileChanger.Size, } fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connID, fileInputData, cmd.contentFile) @@ -179,7 +179,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat connectionID := cmd.fileChanger.ConnectionID if cmd.thumbHeader != nil { defer cmd.thumbFile.Close() - thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.LookupHash} + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, LookupHash: cmd.fileChanger.LookupHash} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 3c4330d48..a59fc6ce5 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -76,6 +76,10 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fileChanger.Path)) } + if filepath.Clean(fileChanger.Path) != fileChanger.Path { + return common.NewError("invalid_path", fmt.Sprintf("%v is not a clean path", fileChanger.Path)) + } + if fileChanger.ConnectionID == "" { return common.NewError("invalid_connection", "Invalid connection id") } @@ -123,7 +127,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj ChunkSize: cmd.fileChanger.ChunkSize, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, - FilePathHash: cmd.fileChanger.LookupHash, + LookupHash: cmd.fileChanger.LookupHash, Size: cmd.fileChanger.Size, } fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, fileInputData, cmd.contentFile) @@ -187,7 +191,7 @@ func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat if cmd.thumbHeader != nil { defer cmd.thumbFile.Close() - thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.LookupHash} + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, LookupHash: cmd.fileChanger.LookupHash} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) @@ -221,7 +225,7 @@ func (cmd *UploadFileCommand) AddChange(ctx context.Context) error { connectionInput, _ := cmd.fileChanger.Marshal() cmd.allocationChange.LookupHash = reference.GetReferenceLookup(cmd.fileChanger.AllocationID, cmd.fileChanger.Path) cmd.allocationChange.Input = connectionInput - logging.Logger.Info("AddChange: ", zap.String("connectionID", cmd.allocationChange.ConnectionID)) + logging.Logger.Info("AddChange: ", zap.String("connectionID", cmd.allocationChange.ConnectionID), zap.String("lookupHash", cmd.allocationChange.LookupHash)) return cmd.allocationChange.Create(ctx) } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index ab0f97950..c237fd3b9 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -454,7 +454,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, common.NewErrorf("download_file", "BlockNum or NumBlocks is too large to convert to int") } - fromPreCommit := fileref.IsPrecommit + fromPreCommit := alloc.AllocationVersion == fileref.AllocationVersion if downloadMode == DownloadContentThumb { rbi := &filestore.ReadBlockInput{ AllocationID: alloc.ID, @@ -464,7 +464,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i NumBlocks: int(dr.NumBlocks), IsThumbnail: true, IsPrecommit: fromPreCommit, - FilestoreVersion: fileref.FilestoreVersion, + FilestoreVersion: filestore.VERSION, } logging.Logger.Info("calling GetFileBlock for thumb", zap.Any("rbi", rbi)) @@ -481,7 +481,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i NumBlocks: int(dr.NumBlocks), VerifyDownload: dr.VerifyDownload, IsPrecommit: fromPreCommit, - FilestoreVersion: fileref.FilestoreVersion, + FilestoreVersion: filestore.VERSION, } logging.Logger.Info("calling GetFileBlock", zap.Any("rbi", rbi)) fileDownloadResponse, err = filestore.GetFileStore().GetFileBlock(rbi) @@ -617,13 +617,12 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b "Invalid connection id. Connection id was not found: %v", err) } if len(connectionObj.Changes) == 0 { - logging.Logger.Info("commit_write", zap.String("connection_id", connectionID)) - if connectionObj.Status == allocation.NewConnection { - return nil, common.NewError("invalid_parameters", - "Invalid connection id. Connection not found.") - } - return nil, common.NewError("invalid_parameters", - "Invalid connection id. Connection does not have any changes.") + logging.Logger.Info("commit_write_empty", zap.String("connection_id", connectionID)) + } + + if len(connectionObj.Changes) > config.Configuration.MaxConnectionChanges { + return nil, common.NewError("max_connection_changes", + "Max connection changes reached. A connection can only have "+fmt.Sprintf("%v", config.Configuration.MaxConnectionChanges)+" changes") } elapsedGetConnObj := time.Since(startTime) - elapsedAllocation - elapsedGetLock @@ -638,7 +637,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } var result blobberhttp.CommitResult - fileIDMeta := make(map[string]string, 0) versionMarkerStr := r.FormValue("version_marker") if versionMarkerStr == "" { return nil, common.NewError("invalid_parameters", "Invalid version marker passed") @@ -655,22 +653,25 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } // Move preCommitDir to finalDir - err = connectionObj.MoveToFilestore(ctx) - if err != nil { - return nil, common.NewError("move_to_filestore_error", fmt.Sprintf("Error while moving to filestore: %s", err.Error())) + if allocationObj.IsRedeemRequired { + err = connectionObj.MoveToFilestore(ctx, allocationObj.AllocationVersion) + if err != nil { + return nil, common.NewError("move_to_filestore_error", fmt.Sprintf("Error while moving to filestore: %s", err.Error())) + } } elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - - err = connectionObj.ApplyChanges( - ctx, common.Now(), fileIDMeta) - if err != nil { - Logger.Error("Error applying changes", zap.Error(err)) - return nil, err + if len(connectionObj.Changes) > 0 { + err = connectionObj.ApplyChanges( + ctx, common.Now(), versionMarker.Version) + if err != nil { + Logger.Error("Error applying changes", zap.Error(err)) + return nil, err + } } elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - elapsedGetConnObj + elapsedGetConnObj - elapsedMoveToFilestore db := datastore.GetStore().GetTransaction(ctx) err = db.Create(&versionMarker).Error @@ -682,17 +683,22 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b allocationObj.BlobberSizeUsed += connectionObj.Size allocationObj.UsedSize += connectionObj.Size allocationObj.AllocationVersion = versionMarker.Version + if len(connectionObj.Changes) == 0 { + allocationObj.IsRedeemRequired = false + } else { + allocationObj.IsRedeemRequired = true + } updateMap := map[string]interface{}{ "used_size": allocationObj.UsedSize, "blobber_size_used": allocationObj.BlobberSizeUsed, - "is_redeem_required": true, + "is_redeem_required": allocationObj.IsRedeemRequired, "allocation_version": versionMarker.Version, "prev_used_size": allocationObj.PrevUsedSize, "prev_blobber_size_used": allocationObj.PrevBlobberSizeUsed, } updateOption := func(a *allocation.Allocation) { - a.IsRedeemRequired = true + a.IsRedeemRequired = allocationObj.IsRedeemRequired a.BlobberSizeUsed = allocationObj.BlobberSizeUsed a.UsedSize = allocationObj.UsedSize a.AllocationVersion = allocationObj.AllocationVersion @@ -721,8 +727,17 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b result.AllocationRoot = allocationObj.AllocationRoot result.Success = true result.ErrorMessage = "" - commitOperation := connectionObj.Changes[0].Operation - input := connectionObj.Changes[0].Input + var ( + commitOperation string + input string + ) + if len(connectionObj.Changes) > 0 { + commitOperation = connectionObj.Changes[0].Operation + input = connectionObj.Changes[0].Input + } else { + commitOperation = "[commit]empty" + input = "[commit]empty" + } //Delete connection object and its changes @@ -774,7 +789,10 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i if new_name == "" { return nil, common.NewError("invalid_parameters", "Invalid name") } - + if filepath.Base(new_name) != new_name { + logging.Logger.Error("invalid_parameters", zap.String("new_name", new_name), zap.String("base", filepath.Base(new_name))) + return nil, common.NewError("invalid_parameters", "Invalid name") + } pathHash, err := pathHashFromReq(r, allocationID) if err != nil { return nil, err @@ -868,7 +886,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int if destPath == "" { return nil, common.NewError("invalid_parameters", "Invalid destination for operation") } - + destPath = filepath.Clean(destPath) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { return nil, err @@ -888,7 +906,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int return nil, common.NewError("meta_error", "Error reading metadata for connection") } - objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "type"}) + objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"}) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) @@ -906,6 +924,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int } } newPath := filepath.Join(destPath, objectRef.Name) + newPath = filepath.Clean(newPath) paths, err := common.GetParentPaths(newPath) if err != nil { return nil, err @@ -936,7 +955,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int allocationChange.LookupHash = pathHash allocationChange.Operation = constants.FileOperationCopy dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ID, - AllocationID: connectionObj.AllocationID, DestPath: destPath, Type: objectRef.Type, SrcPath: objectRef.Path} + AllocationID: connectionObj.AllocationID, DestPath: newPath, Type: objectRef.Type, SrcPath: objectRef.Path} allocation.UpdateConnectionObjSize(connectionID, allocationChange.Size) connectionObj.AddChange(allocationChange, dfc) @@ -983,6 +1002,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int if destPath == "" { return nil, common.NewError("invalid_parameters", "Invalid destination for operation") } + destPath = filepath.Clean(destPath) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { @@ -1023,6 +1043,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int } } newPath := filepath.Join(destPath, objectRef.Name) + newPath = filepath.Clean(newPath) paths, err := common.GetParentPaths(newPath) if err != nil { return nil, err @@ -1056,7 +1077,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int ConnectionID: connectionObj.ID, AllocationID: connectionObj.AllocationID, SrcPath: objectRef.Path, - DestPath: destPath, + DestPath: newPath, Type: objectRef.Type, } dfc.SrcPath = objectRef.Path @@ -1354,6 +1375,10 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } + if !allocationObj.IsRedeemRequired { + return nil, common.NewError("invalid_operation", "Last commit is rollback") + } + versionMarkerString := r.FormValue("version_marker") versionMarker := writemarker.VersionMarker{} err = json.Unmarshal([]byte(versionMarkerString), &versionMarker) @@ -1363,6 +1388,10 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob err) } + if versionMarker.IsRepair { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Rollback marker cannot be a repair marker") + } + var result blobberhttp.CommitResult err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey) if err != nil { @@ -1373,12 +1402,23 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Version marker is same as the current version") } + currentVersionMarker, err := writemarker.GetCurrentVersion(ctx, allocationID) + if err != nil { + return nil, common.NewError("invalid_parameters", "Error getting the current version marker") + } + if currentVersionMarker.IsRepair { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Allocation is in repair mode") + } + if versionMarker.Version != currentVersionMarker.Version-1 { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Version marker is not the previous version") + } + elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock timeoutCtx, cancel := context.WithTimeout(ctx, 45*time.Second) defer cancel() c := datastore.GetStore().CreateTransaction(timeoutCtx) txn := datastore.GetStore().GetTransaction(c) - err = allocation.ApplyRollback(c, allocationID) + err = allocation.ApplyRollback(c, allocationID, allocationObj.AllocationVersion) if err != nil { txn.Rollback() return nil, common.NewError("allocation_rollback_error", "Error applying the rollback for allocation: "+err.Error()) @@ -1394,12 +1434,12 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob alloc.BlobberSizeUsed = alloc.PrevBlobberSizeUsed alloc.UsedSize = alloc.PrevUsedSize - alloc.IsRedeemRequired = true + alloc.IsRedeemRequired = false alloc.AllocationVersion = versionMarker.Version updateMap := map[string]interface{}{ "blobber_size_used": alloc.BlobberSizeUsed, "used_size": alloc.UsedSize, - "is_redeem_required": true, + "is_redeem_required": false, "allocation_version": versionMarker.Version, } @@ -1410,7 +1450,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob a.FileMetaRoot = alloc.FileMetaRoot a.IsRedeemRequired = alloc.IsRedeemRequired } - err = txn.Create(versionMarker).Error + err = txn.Create(&versionMarker).Error if err != nil { txn.Rollback() return &result, common.NewError("write_marker_error", "Error persisting the write marker "+err.Error()) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 95eb0bb89..10765c829 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -263,11 +263,6 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i if err != nil { return nil, common.NewError("bad_db_operation", "Error retrieving file stats. "+err.Error()) } - wm, _ := writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) - if wm != nil && fileStats != nil { - fileStats.WriteMarkerRedeemTxn = wm.CloseTxnID - fileStats.OnChain = wm.OnChain() - } statsMap := make(map[string]interface{}) statsBytes, err := json.Marshal(fileStats) if err != nil { @@ -902,13 +897,16 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb pageLimit = o } } - + var offsetTime int offsetPath := r.FormValue("offsetPath") offsetDate := r.FormValue("offsetDate") updatedDate := r.FormValue("updatedDate") err = checkValidDate(offsetDate, OffsetDateLayout) if err != nil { - return nil, err + offsetTime, err = strconv.Atoi(offsetDate) + if err != nil { + return nil, err + } } err = checkValidDate(updatedDate, OffsetDateLayout) if err != nil { @@ -937,7 +935,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb switch { case refType == "regular": refs, totalPages, newOffsetPath, err = reference.GetRefs( - ctx, allocationID, path, offsetPath, fileType, level, pageLimit, pathRef, + ctx, allocationID, path, offsetPath, fileType, level, pageLimit, offsetTime, pathRef, ) case refType == "updated": diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index 8e5f59441..98454666b 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -2,62 +2,124 @@ package reference import ( "context" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) type QueryCollector interface { CreateRefRecord(ref *Ref) DeleteRefRecord(ref *Ref) - Finalize(ctx context.Context) error + Finalize(ctx context.Context, allocationID string, allocationVersion int64) error AddToCache(ref *Ref) GetFromCache(lookupHash string) *Ref + DeleteLookupRefRecord(ref *Ref) + LockTransaction() + UnlockTransaction() } type dbCollector struct { createdRefs []*Ref deletedRefs []*Ref + refCache RefCache refMap map[string]*Ref + txnLock sync.Mutex + lock sync.Mutex } +type RefCache struct { + AllocationVersion int64 + CreatedRefs []*Ref + DeletedRefs []*Ref +} + +var ( + cacheMap = make(map[string]*RefCache) +) + func NewCollector(changes int) QueryCollector { return &dbCollector{ - createdRefs: make([]*Ref, 0, changes*4), - deletedRefs: make([]*Ref, 0, changes*4), - refMap: make(map[string]*Ref), + createdRefs: make([]*Ref, 0, changes*2), + deletedRefs: make([]*Ref, 0, changes*2), + refCache: RefCache{ + CreatedRefs: make([]*Ref, 0, changes), + DeletedRefs: make([]*Ref, 0, changes), + }, + refMap: make(map[string]*Ref), } } func (dc *dbCollector) CreateRefRecord(ref *Ref) { + dc.lock.Lock() dc.createdRefs = append(dc.createdRefs, ref) + if ref.Type == FILE { + dc.refCache.CreatedRefs = append(dc.refCache.CreatedRefs, ref) + } + dc.lock.Unlock() } func (dc *dbCollector) DeleteRefRecord(ref *Ref) { + dc.lock.Lock() dc.deletedRefs = append(dc.deletedRefs, ref) + if ref.Type == FILE { + dc.refCache.DeletedRefs = append(dc.refCache.DeletedRefs, ref) + } + dc.lock.Unlock() +} + +func (dc *dbCollector) DeleteLookupRefRecord(ref *Ref) { + dc.refCache.DeletedRefs = append(dc.refCache.DeletedRefs, ref) } -func (dc *dbCollector) Finalize(ctx context.Context) error { +func (dc *dbCollector) Finalize(ctx context.Context, allocationID string, allocationVersion int64) error { db := datastore.GetStore().GetTransaction(ctx) if len(dc.deletedRefs) > 0 { - err := db.Delete(dc.deletedRefs).Error + err := db.Delete(&(dc.deletedRefs)).Error if err != nil { return err } } if len(dc.createdRefs) > 0 { - err := db.Create(dc.createdRefs).Error + err := db.Create(&(dc.createdRefs)).Error if err != nil { + for ind, ref := range dc.createdRefs { + logging.Logger.Error("create_ref_error", zap.String("lookup_hash", ref.LookupHash), zap.String("path", ref.Path), zap.Int("index", ind), zap.Int64("allocation_version", allocationVersion)) + } return err } } - + dc.refCache.AllocationVersion = allocationVersion + cacheMap[allocationID] = &(dc.refCache) + logging.Logger.Info("Finalize", zap.Int("created", len(dc.createdRefs)), zap.Int("deleted", len(dc.deletedRefs)), zap.Int64("allocation_version", cacheMap[allocationID].AllocationVersion), zap.String("allocation_id", allocationID)) return nil } func (dc *dbCollector) AddToCache(ref *Ref) { + dc.lock.Lock() dc.refMap[ref.LookupHash] = ref + dc.lock.Unlock() } func (dc *dbCollector) GetFromCache(lookupHash string) *Ref { + dc.lock.Lock() + defer dc.lock.Unlock() return dc.refMap[lookupHash] } + +func GetRefCache(allocationID string) *RefCache { + return cacheMap[allocationID] +} + +func DeleteRefCache(allocationID string) { + cacheMap[allocationID] = nil +} + +func (dc *dbCollector) LockTransaction() { + dc.txnLock.Lock() +} + +func (dc *dbCollector) UnlockTransaction() { + dc.txnLock.Unlock() +} diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index aa6e98df7..f0a721ed0 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -10,10 +10,10 @@ import ( "go.uber.org/zap" ) -func DeleteObject(ctx context.Context, allocationID, lookupHash, _type string, ts common.Timestamp) error { +func DeleteObject(ctx context.Context, allocationID, lookupHash, _type string, ts common.Timestamp, allocationVersion int64, collector QueryCollector) error { db := datastore.GetStore().GetTransaction(ctx) if _type == DIRECTORY { - ref, err := GetLimitedRefFieldsByLookupHashWith(ctx, allocationID, lookupHash, []string{"id"}) + ref, err := GetLimitedRefFieldsByLookupHashWith(ctx, allocationID, lookupHash, []string{"id", "type"}) if err != nil { logging.Logger.Error("delete_object_error", zap.Error(err)) return err @@ -26,8 +26,9 @@ func DeleteObject(ctx context.Context, allocationID, lookupHash, _type string, t if !isEmpty { return common.NewError("invalid_operation", "Directory is not empty") } + _type = ref.Type } - err := db.Exec("UPDATE reference_objects SET is_precommit=?,deleted_at=? WHERE lookup_hash=?", true, sql.NullTime{ + err := db.Exec("UPDATE reference_objects SET deleted_at=? WHERE lookup_hash=?", sql.NullTime{ Time: common.ToTime(ts), Valid: true, }, lookupHash).Error @@ -35,5 +36,11 @@ func DeleteObject(ctx context.Context, allocationID, lookupHash, _type string, t logging.Logger.Error("delete_object_error", zap.Error(err)) return err } + if _type == FILE { + deletedRef := &Ref{ + LookupHash: lookupHash, + } + collector.DeleteLookupRefRecord(deletedRef) + } return err } diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index e8046dee6..1ede74a6c 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -2,6 +2,7 @@ package reference import ( "context" + "database/sql" "fmt" "math" "path/filepath" @@ -75,15 +76,14 @@ type Ref struct { UpdatedAt common.Timestamp `gorm:"column:updated_at;index:idx_updated_at,sort:desc;" dirlist:"updated_at" filelist:"updated_at"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion - IsPrecommit bool `gorm:"column:is_precommit;not null;default:false" filelist:"is_precommit" dirlist:"is_precommit"` ChunkSize int64 `gorm:"column:chunk_size;not null;default:65536" dirlist:"chunk_size" filelist:"chunk_size"` NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"` NumBlockDownloads int64 `gorm:"column:num_of_block_downloads" json:"num_of_block_downloads"` FilestoreVersion int `gorm:"column:filestore_version" json:"-"` DataHash string `gorm:"column:data_hash" filelist:"data_hash"` DataHashSignature string `gorm:"column:data_hash_signature" filelist:"data_hash_signature"` + AllocationVersion int64 `gorm:"allocation_version" dirlist:"allocation_version" filelist:"allocation_version"` IsEmpty bool `gorm:"-" dirlist:"is_empty"` - AllocationVersion int64 `gorm:"-" dirlist:"allocation_version" filelist:"allocation_version"` HashToBeComputed bool `gorm:"-"` prevID int64 `gorm:"-"` } @@ -146,53 +146,67 @@ func GetReferenceLookup(allocationID, path string) string { } func NewDirectoryRef() *Ref { - return &Ref{Type: DIRECTORY, IsPrecommit: true} + return &Ref{Type: DIRECTORY} } func NewFileRef() *Ref { - return &Ref{Type: FILE, IsPrecommit: true} + return &Ref{Type: FILE} } // Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child -func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timestamp, collector QueryCollector) (*Ref, error) { +func Mkdir(ctx context.Context, allocationID, destpath string, allocationVersion int64, ts common.Timestamp, collector QueryCollector) (*Ref, error) { + var err error db := datastore.GetStore().GetTransaction(ctx) if destpath != "/" { destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") } destLookupHash := GetReferenceLookup(allocationID, destpath) - var destRef Ref + var destRef *Ref cachedRef := collector.GetFromCache(destLookupHash) if cachedRef != nil { - destRef = *cachedRef + destRef = cachedRef } else { - err := db.Select("id", "type").Where(&Ref{LookupHash: destLookupHash}).Take(&destRef).Error + destRef, err = GetReferenceByLookupHashWithNewTransaction(destLookupHash) if err != nil && err != gorm.ErrRecordNotFound { return nil, err } - destRef.LookupHash = destLookupHash - if destRef.ID > 0 { - defer collector.AddToCache(&destRef) + if destRef != nil { + destRef.LookupHash = destLookupHash + defer collector.AddToCache(destRef) } } - if destRef.ID > 0 { + if destRef != nil { if destRef.Type != DIRECTORY { return nil, common.NewError("invalid_dir_tree", "parent path is not a directory") } - return &destRef, nil + return destRef, nil } fields, err := common.GetAllParentPaths(destpath) if err != nil { logging.Logger.Error("mkdir: failed to get all parent paths", zap.Error(err), zap.String("destpath", destpath)) return nil, err } - tx := db.Model(&Ref{}).Select("id", "path", "type") parentLookupHashes := make([]string, 0, len(fields)) for i := 0; i < len(fields); i++ { parentLookupHashes = append(parentLookupHashes, GetReferenceLookup(allocationID, fields[i])) - tx = tx.Or(Ref{LookupHash: parentLookupHashes[i]}) } - var parentRefs []*Ref + collector.LockTransaction() + defer collector.UnlockTransaction() + cachedRef = collector.GetFromCache(destLookupHash) + if cachedRef != nil { + if cachedRef.Type != DIRECTORY { + return nil, common.NewError("invalid_dir_tree", "parent path is not a directory") + } + return cachedRef, nil + } else { + logging.Logger.Info("noEntryFound: ", zap.String("destLookupHash", destLookupHash), zap.String("destpath", destpath)) + } + + tx := db.Model(&Ref{}).Select("id", "path", "type") + for i := 0; i < len(fields); i++ { + tx = tx.Or(Ref{LookupHash: parentLookupHashes[i]}) + } err = tx.Order("path").Find(&parentRefs).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err @@ -202,7 +216,6 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta parentPath = "/" ) if len(parentRefs) > 0 { - logging.Logger.Info("mkdir:parentRefs: ", zap.Any("parentRefs", parentRefs)) parentID = parentRefs[len(parentRefs)-1].ID parentPath = parentRefs[len(parentRefs)-1].Path for i := 0; i < len(parentRefs); i++ { @@ -218,8 +231,9 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta fields = append(fields, destpath) parentLookupHashes = append(parentLookupHashes, destLookupHash) } + for i := len(parentRefs); i < len(fields); i++ { - logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Any("parentID", parentID), zap.Int("pathLevel", i+1)) + logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Int("parentID", int(parentID))) var parentIDRef *int64 if parentID > 0 { parentIDRef = &parentID @@ -238,11 +252,13 @@ func Mkdir(ctx context.Context, allocationID, destpath string, ts common.Timesta newRef.LookupHash = parentLookupHashes[i] newRef.CreatedAt = ts newRef.UpdatedAt = ts - newRef.FileMetaHash = encryption.Hash(newRef.GetFileMetaHashData()) + newRef.FileMetaHash = encryption.FastHash(newRef.GetFileMetaHashData()) + newRef.AllocationVersion = allocationVersion err = db.Create(newRef).Error if err != nil { return nil, err } + collector.AddToCache(newRef) parentID = newRef.ID parentPath = newRef.Path } @@ -614,17 +630,8 @@ func (r *Ref) UpdatePath(newPath, parentPath string) { r.LookupHash = GetReferenceLookup(r.AllocationID, r.Path) } -func DeleteReference(ctx context.Context, refID int64, pathHash string) error { - if refID <= 0 { - return common.NewError("invalid_ref_id", "Invalid reference ID to delete") - } - db := datastore.GetStore().GetTransaction(ctx) - return db.Where("path_hash = ?", pathHash).Delete(&Ref{ID: refID}).Error -} - func (r *Ref) SaveFileRef(ctx context.Context, collector QueryCollector) error { r.prevID = r.ID - r.IsPrecommit = true r.NumUpdates += 1 if r.ID > 0 { deleteRef := &Ref{ID: r.ID} @@ -638,7 +645,6 @@ func (r *Ref) SaveFileRef(ctx context.Context, collector QueryCollector) error { func (r *Ref) SaveDirRef(ctx context.Context, collector QueryCollector) error { r.prevID = r.ID - r.IsPrecommit = true r.NumUpdates += 1 if r.ID > 0 { deleteRef := &Ref{ID: r.ID} @@ -752,3 +758,31 @@ func IsDirectoryEmpty(ctx context.Context, id int64) (bool, error) { return true, nil } + +func GetReferenceByLookupHashWithNewTransaction(lookupHash string) (*Ref, error) { + var ref *Ref + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + txn := datastore.GetStore().GetTransaction(ctx) + return txn.Model(&Ref{}).Select("id", "type").Where("lookup_hash = ?", lookupHash).Take(&ref).Error + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return nil, err + } + return ref, nil +} + +func GetFullReferenceByLookupHashWithNewTransaction(lookupHash string) (*Ref, error) { + var ref *Ref + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + txn := datastore.GetStore().GetTransaction(ctx) + return txn.Model(&Ref{}).Where("lookup_hash = ?", lookupHash).Take(&ref).Error + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return nil, err + } + return ref, nil +} diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index ccb7929bf..16b0e6884 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -238,7 +238,7 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) // Might need to consider covering index for efficient search https://blog.crunchydata.com/blog/why-covering-indexes-are-incredibly-helpful // To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs // very easily and effectively; Same case for offsetDate. -func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { +func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit, offsetTime int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { var ( pRefs = make([]PaginatedRef, 0, pageLimit/4) dbError error @@ -246,13 +246,22 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, ) path = filepath.Clean(path) tx := datastore.GetStore().GetTransaction(ctx) - pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + 1 - if pathLevel == level { + pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) { + pRefs = append(pRefs, *parentRef) + refs = &pRefs + newOffsetPath = parentRef.Path + } + + if pathLevel+1 == level { dbQuery = tx.Model(&Ref{}).Where("parent_id = ?", parentRef.ID) if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } dbQuery = dbQuery.Where("path > ?", offsetPath) + if offsetTime != 0 { + dbQuery = dbQuery.Where("created_at < ?", offsetTime) + } dbQuery = dbQuery.Order("path") } else { dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"%") @@ -262,6 +271,9 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, if level != 0 { dbQuery = dbQuery.Where("level = ?", level) } + if offsetTime != 0 { + dbQuery = dbQuery.Where("created_at < ?", offsetTime) + } dbQuery = dbQuery.Where("path > ?", offsetPath) diff --git a/code/go/0chain.net/core/encryption/hash.go b/code/go/0chain.net/core/encryption/hash.go index 53f649343..b6859cb1b 100644 --- a/code/go/0chain.net/core/encryption/hash.go +++ b/code/go/0chain.net/core/encryption/hash.go @@ -3,6 +3,8 @@ package encryption import ( "crypto/sha1" "encoding/hex" + "hash" + "sync" "github.com/minio/sha256-simd" "golang.org/x/crypto/sha3" @@ -10,6 +12,12 @@ import ( const HASH_LENGTH = 32 +var sha3Pool = sync.Pool{ + New: func() interface{} { + return sha3.New256() + }, +} + type HashBytes [HASH_LENGTH]byte /*Hash - hash the given data and return the hash as hex string */ @@ -30,9 +38,12 @@ func RawHash(data interface{}) []byte { default: panic("unknown type") } - hash := sha3.New256() + hash := sha3Pool.Get().(hash.Hash) hash.Write(databuf) - return hash.Sum(nil) + res := hash.Sum(nil) + hash.Reset() + sha3Pool.Put(hash) + return res } func ShaHash(data interface{}) []byte { diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index eac82db9c..5dddcffb0 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -79,6 +79,8 @@ rate_limiters: commit_limit_daily: 1600 # Max commit limit with size zero or less in a day for a client. Default is 400 commit_zero_limit_daily: 400 + # Max connection changes in a batch. Default is 100 + max_connection_changes: 100 server_chain: id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" diff --git a/go.mod b/go.mod index 88c0a0dfe..6c6cdaac4 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/selvatico/go-mocket v1.0.7 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 - go.uber.org/zap v1.24.0 + go.uber.org/zap v1.27.0 golang.org/x/crypto v0.17.0 golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 @@ -33,12 +33,12 @@ require ( gorm.io/datatypes v1.2.0 gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.5 + gorm.io/gorm v1.25.11 ) require ( github.com/lithammer/shortuuid/v3 v3.0.7 - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.7.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc ) @@ -53,6 +53,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.26.1 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.5 github.com/pressly/goose/v3 v3.13.4 + moul.io/zapgorm2 v1.3.0 ) require ( @@ -148,9 +149,8 @@ require ( go.dedis.ch/fixbuf v1.0.3 // indirect go.dedis.ch/kyber/v3 v3.1.0 // indirect go.mongodb.org/mongo-driver v1.11.3 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/text v0.16.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index 2980a6cc5..8294050d9 100644 --- a/go.sum +++ b/go.sum @@ -107,7 +107,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNIC github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -840,6 +839,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= @@ -870,23 +870,24 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -942,8 +943,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -987,6 +989,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1013,8 +1016,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1073,9 +1076,11 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1094,8 +1099,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1164,8 +1169,9 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1330,9 +1336,10 @@ gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/driver/sqlserver v1.4.1/go.mod h1:DJ4P+MeZbc5rvY58PnmN1Lnyvb5gw5NPzGshHDnJLig= +gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= -gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1361,6 +1368,8 @@ modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk= +moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 6d4cf2da3..9496a21b4 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -357,13 +357,13 @@ CREATE TABLE reference_objects ( created_at bigint, updated_at bigint, deleted_at timestamp with time zone, - is_precommit boolean DEFAULT false NOT NULL, chunk_size bigint DEFAULT 65536 NOT NULL, num_of_updates bigint, num_of_block_downloads bigint, data_hash character varying(64), data_hash_signature character varying(64), - parent_id bigint DEFAULT NULL + parent_id bigint DEFAULT NULL, + allocation_version bigint DEFAULT 0 NOT NULL ); @@ -572,8 +572,6 @@ ALTER TABLE ONLY marketplace_share_info ALTER COLUMN id SET DEFAULT nextval('mar ALTER TABLE ONLY reference_objects ALTER COLUMN id SET DEFAULT nextval('reference_objects_id_seq'::regclass); --- ALTER TABLE ONLY reference_objects ADD CONSTRAINT path_commit UNIQUE(lookup_hash,is_precommit); - -- -- Name: terms id; Type: DEFAULT; Schema: public; Owner: blobber_user -- @@ -756,7 +754,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash) INCLUDE(id,type) where deleted_at IS NULL; +CREATE UNIQUE INDEX idx_lookup_hash_deleted ON reference_objects USING btree (lookup_hash,(deleted_at IS NULL)) INCLUDE(id,type,num_of_updates); -- @@ -787,17 +785,17 @@ CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english': -- CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL; -- --- Name: idx_parent_id_alloc; Type: INDEX; Schema: public; Owner: blobber_user +-- Name: idx_parent_id; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_parent_id_alloc ON reference_objects USING btree (parent_id) where deleted_at is NULL; +CREATE INDEX idx_parent_id ON reference_objects USING btree (parent_id); -- -- Name: idx_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path) include(id,level,type) WHERE deleted_at IS NULL; +CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path) WHERE deleted_at IS NULL; -- @@ -865,14 +863,6 @@ ALTER TABLE ONLY allocation_changes CREATE INDEX connection_id_index ON allocation_changes USING btree (connection_id); --- --- Name: file_stats fk_file_stats_ref; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user --- - -ALTER TABLE ONLY file_stats - ADD CONSTRAINT fk_file_stats_ref FOREIGN KEY (ref_id) REFERENCES reference_objects(id) ON DELETE CASCADE; - - -- -- Name: terms fk_terms_allocation; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user -- diff --git a/goose/migrations/1717416291_change_lookuphash.sql b/goose/migrations/1717416291_change_lookuphash.sql index e2d8e71dd..f55ac0827 100644 --- a/goose/migrations/1717416291_change_lookuphash.sql +++ b/goose/migrations/1717416291_change_lookuphash.sql @@ -1,6 +1,6 @@ -- +goose Up -- +goose StatementBegin -ALTER TABLE allocation_changes ADD COLUMN lookup_hash character varying(64); +ALTER TABLE allocation_changes ADD COLUMN lookup_hash character varying(64); -- CREATE UNIQUE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash,connection_id); -- +goose StatementEnd \ No newline at end of file diff --git a/goose/migrations/1718188301_change_idx.sql b/goose/migrations/1718188301_change_idx.sql index 564e37891..14e6c3ba0 100644 --- a/goose/migrations/1718188301_change_idx.sql +++ b/goose/migrations/1718188301_change_idx.sql @@ -1,5 +1,10 @@ -- +goose Up -- +goose StatementBegin -CREATE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash); + -- + -- Name: connection_id_lookup_hash; Type: UNIQUE CONSTRAINT; Schema: public; Owner: blobber_user + -- + +ALTER TABLE ONLY allocation_changes ADD CONSTRAINT connection_id_lookup_hash UNIQUE(connection_id,lookup_hash); + -- +goose StatementEnd \ No newline at end of file diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index 20cc6ea50..6890115ea 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -1,8 +1,10 @@ -- +goose Up -- +goose StatementBegin -DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; +DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin; -CREATE INDEX idx_is_precommit_deleted_at on reference_objects(allocation_id) INCLUDE(type,lookup_hash,id,type) WHERE is_precommit=true AND deleted_at IS NULL; +CREATE INDEX idx_is_allocation_version_deleted_at on reference_objects(allocation_id,allocation_version) WHERE type='f' AND deleted_at IS NULL; CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL; + +CREATE INDEX idx_path_alloc_level ON reference_objects USING btree (allocation_id,level,type,path) WHERE deleted_at IS NULL; -- +goose StatementEnd \ No newline at end of file From 5a252665840e69ef49c11baf539ded7fecdc3a16 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sun, 11 Aug 2024 02:22:53 +0530 Subject: [PATCH 66/98] add size for root dir --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index fa2a303ed..2e880ec9f 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -405,6 +405,9 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } fileref = parent } + if path == "/" { + fileref.Size = allocationObj.BlobberSizeUsed + } listResult.Meta = fileref.GetListingData(ctx) if clientID != allocationObj.OwnerID { delete(listResult.Meta, "path") @@ -473,6 +476,9 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } dirref = r + if path == "/" { + dirref.Size = allocationObj.BlobberSizeUsed + } } var result blobberhttp.ListResult From d74880965629ed457110142e5ea282044a371238 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 19 Aug 2024 01:03:19 +0530 Subject: [PATCH 67/98] rmv hash len check in get path for file --- code/go/0chain.net/blobbercore/filestore/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index c9cdf171a..611157476 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -846,7 +846,7 @@ func (fs *FileStore) getAllocDir(allocID string) string { } func (fs *FileStore) GetPathForFile(allocID, hash string, version int) (string, error) { - if len(allocID) != 64 || len(hash) != 64 { + if len(allocID) != 64 { return "", errors.New("length of allocationID/hash must be 64") } var versionStr string From dd6787fe08fa271a9f88d7ccf0a2285f6cdcae6d Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 6 Sep 2024 14:41:29 +0530 Subject: [PATCH 68/98] add slash seperator --- code/go/0chain.net/blobbercore/reference/referencepath.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 16b0e6884..f9566a1b8 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -264,7 +264,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, } dbQuery = dbQuery.Order("path") } else { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"%") + dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"/%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } From 350bb7ef03ab7b751c7273ac2a282c44f7d67827 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Tue, 24 Sep 2024 16:21:24 +0530 Subject: [PATCH 69/98] fix list root --- code/go/0chain.net/blobbercore/reference/referencepath.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index f9566a1b8..8972d50ec 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -264,7 +264,11 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, } dbQuery = dbQuery.Order("path") } else { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"/%") + listPath := path + if path != "/" { + listPath = path + "/" + } + dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } From f8aefee12d0ab381fe25452876120dbed75246de Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 25 Sep 2024 00:21:14 +0530 Subject: [PATCH 70/98] add log for connection obj --- code/go/0chain.net/blobbercore/allocation/connection.go | 2 ++ code/go/0chain.net/blobbercore/handler/worker.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index 75ac30b0c..47e418712 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -246,6 +246,7 @@ func GetHasher(connectionID, pathHash string) *filestore.CommitHasher { // DeleteConnectionObjEntry remove the connectionID entry from map // If the given connectionID is not present, then it is no-op. func DeleteConnectionObjEntry(connectionID string) { + logging.Logger.Info("DeleteConnectionObjEntry", zap.String("connection_id", connectionID)) connectionObjMutex.Lock() connectionObj, ok := connectionProcessor[connectionID] if ok { @@ -262,6 +263,7 @@ func cleanConnectionObj() { for connectionID, connectionObj := range connectionProcessor { diff := time.Since(connectionObj.UpdatedAt) if diff >= ConnectionObjTimeout { + logging.Logger.Info("cleanConnectionObj", zap.String("connection_id", connectionID), zap.Duration("diff", diff)) // Stop the context and hash worker connectionObj.cnclCtx() for _, change := range connectionObj.changes { diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 19cf49c30..b5acbe174 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -87,6 +87,10 @@ func cleanupTempFiles(ctx context.Context) { for i := 0; i < len(openConnectionsToDelete); i++ { connection := &openConnectionsToDelete[i] logging.Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ID)) + processor := allocation.GetConnectionProcessor(connection.ID) + if processor != nil { + continue + } connection.ComputeProperties() nctx := datastore.GetStore().CreateTransaction(ctx) From ffb861c68cb877d8cb8f9c2dfd34bd3e2f641cef Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 27 Sep 2024 13:52:52 +0530 Subject: [PATCH 71/98] add log for blacklist --- code/go/0chain.net/blobbercore/handler/client_quota.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/client_quota.go b/code/go/0chain.net/blobbercore/handler/client_quota.go index cee45a830..4335192d4 100644 --- a/code/go/0chain.net/blobbercore/handler/client_quota.go +++ b/code/go/0chain.net/blobbercore/handler/client_quota.go @@ -8,6 +8,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/gorm" ) @@ -84,6 +86,7 @@ func AddWriteMarkerCount(clientID string, zeroSizeWM bool) { cs.TotalZeroWM++ } if cs.TotalZeroWM > config.Configuration.CommitZeroLimitDaily || cs.TotalWM > config.Configuration.CommitLimitDaily { + logging.Logger.Info("Client blacklisted", zap.String("client_id", clientID), zap.Int64("total_write_marker", cs.TotalWM), zap.Int64("total_zero_write_marker", cs.TotalZeroWM), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily), zap.Int64("commit_zero_limit_daily", config.Configuration.CommitZeroLimitDaily)) SetBlacklist(clientID) } } @@ -136,6 +139,7 @@ func saveClientStats() { } func startBlackListWorker(ctx context.Context) { + logging.Logger.Info("Starting black list worker", zap.Int64("upload_limit", config.Configuration.UploadLimitMonthly), zap.Int64("download_limit", config.Configuration.BlockLimitMonthly), zap.Int64("commit_limit", config.Configuration.CommitLimitMonthly), zap.Int64("commit_zero_limit", config.Configuration.CommitZeroLimitDaily), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily)) BlackListWorkerTime := 24 * time.Hour if config.Development() { BlackListWorkerTime = 10 * time.Second From dfd6a266b6985f79e3d6a927aac337b48e255e27 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 30 Sep 2024 20:53:43 +0530 Subject: [PATCH 72/98] add get refs log --- code/go/0chain.net/blobbercore/reference/referencepath.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 8972d50ec..d044b6465 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -247,7 +247,9 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, path = filepath.Clean(path) tx := datastore.GetStore().GetTransaction(ctx) pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + logging.Logger.Info("getRefs: CheckSingleRef", zap.Int("pathLevel", pathLevel), zap.Int("level", level), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("pageLimit", pageLimit)) if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) { + logging.Logger.Info("getRefs: SingleRef") pRefs = append(pRefs, *parentRef) refs = &pRefs newOffsetPath = parentRef.Path @@ -268,6 +270,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, if path != "/" { listPath = path + "/" } + logging.Logger.Info("getRefs: ListRef", zap.String("listPath", listPath)) dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) From 991b6388665e8c70de320b0ab4ec934ed54674bb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 30 Sep 2024 21:31:49 +0530 Subject: [PATCH 73/98] add return --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 3 +++ code/go/0chain.net/blobbercore/reference/referencepath.go | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 2e880ec9f..56625bf28 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -943,6 +943,9 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb refs, totalPages, newOffsetPath, err = reference.GetRefs( ctx, allocationID, path, offsetPath, fileType, level, pageLimit, offsetTime, pathRef, ) + if refs != nil { + logging.Logger.Info("GetRefs: regular", zap.Int("refs", len(*refs)), zap.Int("totalPages", totalPages), zap.String("newOffsetPath", newOffsetPath), zap.Error(err)) + } case refType == "updated": refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs( diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index d044b6465..f2bfc7830 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -249,10 +249,10 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) logging.Logger.Info("getRefs: CheckSingleRef", zap.Int("pathLevel", pathLevel), zap.Int("level", level), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("pageLimit", pageLimit)) if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) { - logging.Logger.Info("getRefs: SingleRef") pRefs = append(pRefs, *parentRef) refs = &pRefs newOffsetPath = parentRef.Path + return } if pathLevel+1 == level { @@ -270,7 +270,6 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, if path != "/" { listPath = path + "/" } - logging.Logger.Info("getRefs: ListRef", zap.String("listPath", listPath)) dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) From 37296f38bfade17a06133066fa1276407ed904fb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sun, 27 Oct 2024 21:10:06 +0530 Subject: [PATCH 74/98] fix concurrent read and write --- code/go/0chain.net/blobbercore/reference/dbCollector.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index 98454666b..d9436840b 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -36,7 +36,8 @@ type RefCache struct { } var ( - cacheMap = make(map[string]*RefCache) + cacheMap = make(map[string]*RefCache) + cacheMapLock sync.RWMutex ) func NewCollector(changes int) QueryCollector { @@ -91,8 +92,10 @@ func (dc *dbCollector) Finalize(ctx context.Context, allocationID string, alloca } } dc.refCache.AllocationVersion = allocationVersion + cacheMapLock.Lock() cacheMap[allocationID] = &(dc.refCache) logging.Logger.Info("Finalize", zap.Int("created", len(dc.createdRefs)), zap.Int("deleted", len(dc.deletedRefs)), zap.Int64("allocation_version", cacheMap[allocationID].AllocationVersion), zap.String("allocation_id", allocationID)) + cacheMapLock.Unlock() return nil } @@ -109,11 +112,15 @@ func (dc *dbCollector) GetFromCache(lookupHash string) *Ref { } func GetRefCache(allocationID string) *RefCache { + cacheMapLock.RLock() + defer cacheMapLock.RUnlock() return cacheMap[allocationID] } func DeleteRefCache(allocationID string) { + cacheMapLock.Lock() cacheMap[allocationID] = nil + delete(cacheMap, allocationID) } func (dc *dbCollector) LockTransaction() { From 421e1a46be8eee26371c5465eb104a73b5113543 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sun, 27 Oct 2024 21:12:05 +0530 Subject: [PATCH 75/98] add unlock --- code/go/0chain.net/blobbercore/reference/dbCollector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index d9436840b..5531bcf60 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -120,7 +120,7 @@ func GetRefCache(allocationID string) *RefCache { func DeleteRefCache(allocationID string) { cacheMapLock.Lock() cacheMap[allocationID] = nil - delete(cacheMap, allocationID) + cacheMapLock.Unlock() } func (dc *dbCollector) LockTransaction() { From 66ca94e766749da3ff1ae373d402a1fb399399c4 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Mon, 11 Nov 2024 19:13:42 +0530 Subject: [PATCH 76/98] fix duplicate dir (#1504) * fix duplicate dir * fix duplicate check --- .../blobbercore/allocation/newdirchange.go | 17 ++++++++++++++++- code/go/0chain.net/blobbercore/reference/ref.go | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 6d5778fd0..3db4ba3a4 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -11,6 +11,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/gorm" ) @@ -33,6 +35,19 @@ func (nf *NewDir) ApplyChange(ctx context.Context, if parentRef == nil || parentRef.ID == 0 { _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, allocationVersion, ts, collector) } else { + collector.LockTransaction() + defer collector.UnlockTransaction() + dirLookupHash := reference.GetReferenceLookup(nf.AllocationID, nf.Path) + dRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, nf.AllocationID, dirLookupHash, []string{"id"}) + if err != nil && err != gorm.ErrRecordNotFound { + logging.Logger.Error("ApplyChange:Newdir", zap.Error(err)) + return err + } + err = nil + // already exists + if dRef != nil && dRef.ID != 0 { + return nil + } parentIDRef := &parentRef.ID newRef := reference.NewDirectoryRef() newRef.AllocationID = nf.AllocationID @@ -43,7 +58,7 @@ func (nf *NewDir) ApplyChange(ctx context.Context, newRef.Name = filepath.Base(nf.Path) newRef.PathLevel = len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")) newRef.ParentID = parentIDRef - newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) + newRef.LookupHash = dirLookupHash newRef.CreatedAt = ts newRef.UpdatedAt = ts newRef.FileMetaHash = encryption.FastHash(newRef.GetFileMetaHashData()) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 1068815c6..53df6c7da 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -256,6 +256,7 @@ func Mkdir(ctx context.Context, allocationID, destpath string, allocationVersion newRef.AllocationVersion = allocationVersion err = db.Create(newRef).Error if err != nil { + logging.Logger.Error("mkdir: failed to create directory", zap.Error(err), zap.String("path", fields[i])) return nil, err } collector.AddToCache(newRef) From 6c1952f7a60406fce35418a1b01392c52696204a Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 9 Dec 2024 23:15:13 +0700 Subject: [PATCH 77/98] update limit monthly --- code/go/0chain.net/blobbercore/config/config.go | 2 +- config/0chain_blobber.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 7403ecb52..73c9e3bdf 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -32,7 +32,7 @@ func SetupDefaultConfig() { viper.SetDefault("rate_limiters.block_limit_request", 500) viper.SetDefault("rate_limiters.block_limit_monthly", 31250000) viper.SetDefault("rate_limiters.upload_limit_monthly", 31250000) - viper.SetDefault("rate_limiters.commit_limit_monthly", 30000) + viper.SetDefault("rate_limiters.commit_limit_monthly", 1000000000) viper.SetDefault("rate_limiters.commit_limit_daily", 1600) viper.SetDefault("rate_limiters.commit_zero_limit_daily", 400) viper.SetDefault("rate_limiters.max_connection_changes", 100) diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index 100a3788f..758bab823 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -76,7 +76,7 @@ rate_limiters: # Max upload limit in a month for a client. Default is 2000GB(the value needs to be in blocks which is data/64KB) upload_limit_monthly: 31250000 # Max commit limit in a month for a client. Default is 30000 - commit_limit_monthly: 30000 + commit_limit_monthly: 1000000000 # Max commit limit in a day for a client. Default is 1600 commit_limit_daily: 1600 # Max commit limit with size zero or less in a day for a client. Default is 400 From f69d989363676f7d00767acd6721610bad2c1f3b Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 12 Dec 2024 18:05:01 +0700 Subject: [PATCH 78/98] remove temp size check --- code/go/0chain.net/blobbercore/filestore/state.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/code/go/0chain.net/blobbercore/filestore/state.go b/code/go/0chain.net/blobbercore/filestore/state.go index b098a34c8..f68b556f1 100644 --- a/code/go/0chain.net/blobbercore/filestore/state.go +++ b/code/go/0chain.net/blobbercore/filestore/state.go @@ -57,8 +57,6 @@ func (fs *FileStore) initMap() error { return errors.New("could not get db client") } - limitCh := make(chan struct{}, 50) - wg := &sync.WaitGroup{} var dbAllocations []*dbAllocation err := db.Model(&dbAllocation{}).FindInBatches(&dbAllocations, 1000, func(tx *gorm.DB, batch int) error { @@ -78,18 +76,12 @@ func (fs *FileStore) initMap() error { if err != nil { return err } - - limitCh <- struct{}{} - wg.Add(1) - go fs.getTemporaryStorageDetails(ctx, &a, dbAlloc.ID, limitCh, wg) - } fs.setAllocations(allocsMap) return nil }).Error - wg.Wait() return err }) return err From d2b4c26e0fbd6bd2d1d715de6a029c8442c8721b Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Tue, 7 Jan 2025 04:07:48 +0530 Subject: [PATCH 79/98] Fix auth ticket enterprise --- .../blobbercore/handler/auth_ticket.go | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/auth_ticket.go b/code/go/0chain.net/blobbercore/handler/auth_ticket.go index 89032880c..5a8923b9a 100644 --- a/code/go/0chain.net/blobbercore/handler/auth_ticket.go +++ b/code/go/0chain.net/blobbercore/handler/auth_ticket.go @@ -2,8 +2,10 @@ package handler import ( "context" + "fmt" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/common/core/common" + "github.com/0chain/gosdk/core/encryption" "net/http" ) @@ -19,28 +21,33 @@ type AuthTicketResponse struct { // // parameters: // -// +name: Zbox-Signature -// in: header -// type: string -// description: Digital signature to verify that the sender is 0box service. -// +name: client_id -// type: string -// in: query -// description: Client ID is used as a payload to the token generated. The token represents a signed version of this string by the blobber's private key. +// +name: Zbox-Signature +// in: header +// type: string +// description: Digital signature to verify that the sender is 0box service. +// +name: client_id +// type: string +// in: query +// description: Client ID is used as a payload to the token generated. The token represents a signed version of this string by the blobber's private key. // // responses: -// 200: AuthTicketResponse +// +// 200: AuthTicketResponse func GenerateAuthTicket(ctx context.Context, r *http.Request) (interface{}, error) { + clientID := r.URL.Query().Get("client_id") if clientID == "" { return nil, common.NewError("missing_client_id", "client_id is required") } - signature, err := node.Self.Sign(clientID) + round := r.URL.Query().Get("round") + + payload := encryption.Hash(fmt.Sprintf("%s_%s", clientID, round)) + + signature, err := node.Self.Sign(payload) if err != nil { return nil, common.NewError("signature_failed", "signature failed") } - return &AuthTicketResponse{ AuthTicket: signature, }, nil From 06d5ad9071b163532db0288d5fcebfeb1e669e1a Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Tue, 7 Jan 2025 04:26:45 +0530 Subject: [PATCH 80/98] Debug --- code/go/0chain.net/blobbercore/handler/auth_ticket.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/auth_ticket.go b/code/go/0chain.net/blobbercore/handler/auth_ticket.go index 5a8923b9a..48f973252 100644 --- a/code/go/0chain.net/blobbercore/handler/auth_ticket.go +++ b/code/go/0chain.net/blobbercore/handler/auth_ticket.go @@ -3,9 +3,11 @@ package handler import ( "context" "fmt" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/common/core/common" "github.com/0chain/gosdk/core/encryption" + "go.uber.org/zap" "net/http" ) @@ -41,9 +43,14 @@ func GenerateAuthTicket(ctx context.Context, r *http.Request) (interface{}, erro } round := r.URL.Query().Get("round") + if round == "" { + return nil, common.NewError("missing_round", "round is required") + } payload := encryption.Hash(fmt.Sprintf("%s_%s", clientID, round)) + logging.Logger.Info("GenerateAuthTicket", zap.String("payload", payload), zap.String("client_id", clientID), zap.String("round", round)) + signature, err := node.Self.Sign(payload) if err != nil { return nil, common.NewError("signature_failed", "signature failed") From 68f7733c2c5554db2c43500567878a72509f8081 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 27 Jan 2025 23:35:08 +0530 Subject: [PATCH 81/98] dummy commit --- code/go/0chain.net/blobbercore/allocation/newdirchange.go | 1 + 1 file changed, 1 insertion(+) diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 3db4ba3a4..e5bf4051e 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -32,6 +32,7 @@ func (nf *NewDir) ApplyChange(ctx context.Context, if err != nil && err != gorm.ErrRecordNotFound { return err } + if parentRef == nil || parentRef.ID == 0 { _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, allocationVersion, ts, collector) } else { From d46f49eccda143a754580ca6277ef11bd16aef81 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Wed, 29 Jan 2025 20:57:02 +0530 Subject: [PATCH 82/98] Set enterprise = true always --- code/go/0chain.net/blobbercore/handler/protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go index 976828558..d729fe737 100644 --- a/code/go/0chain.net/blobbercore/handler/protocol.go +++ b/code/go/0chain.net/blobbercore/handler/protocol.go @@ -66,7 +66,7 @@ func getStorageNode() (*transaction.StorageNode, error) { sn.StakePoolSettings.NumDelegates = config.Configuration.NumDelegates sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge - sn.IsEnterprise = config.Configuration.IsEnterprise + sn.IsEnterprise = true return sn, nil } From 2d169e86d72b4ec19de6365a6ecc935bbeda7c22 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Sat, 1 Feb 2025 00:18:16 +0530 Subject: [PATCH 83/98] Revert "Set enterprise = true always" This reverts commit d46f49eccda143a754580ca6277ef11bd16aef81. --- code/go/0chain.net/blobbercore/handler/protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go index d729fe737..976828558 100644 --- a/code/go/0chain.net/blobbercore/handler/protocol.go +++ b/code/go/0chain.net/blobbercore/handler/protocol.go @@ -66,7 +66,7 @@ func getStorageNode() (*transaction.StorageNode, error) { sn.StakePoolSettings.NumDelegates = config.Configuration.NumDelegates sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge - sn.IsEnterprise = true + sn.IsEnterprise = config.Configuration.IsEnterprise return sn, nil } From e2dfba184661ca3c79ad03428880f4dfd7ea3ce2 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Sat, 1 Feb 2025 03:06:49 +0530 Subject: [PATCH 84/98] sv set to 1 --- code/go/0chain.net/blobbercore/handler/protocol.go | 1 + code/go/0chain.net/core/transaction/entity.go | 1 + 2 files changed, 2 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go index 976828558..8a5993e8b 100644 --- a/code/go/0chain.net/blobbercore/handler/protocol.go +++ b/code/go/0chain.net/blobbercore/handler/protocol.go @@ -67,6 +67,7 @@ func getStorageNode() (*transaction.StorageNode, error) { sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge sn.IsEnterprise = config.Configuration.IsEnterprise + sn.StorageVersion = 1 return sn, nil } diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go index 6a41ff980..00908352c 100644 --- a/code/go/0chain.net/core/transaction/entity.go +++ b/code/go/0chain.net/core/transaction/entity.go @@ -73,6 +73,7 @@ type StorageNode struct { PublicKey string `json:"-"` StakePoolSettings StakePoolSettings `json:"stake_pool_settings"` IsEnterprise bool `json:"is_enterprise"` + StorageVersion int `json:"storage_version"` } type BlobberAllocation struct { From 3d083358b4ababd00ac5e9739502f4f5382233e8 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 5 Feb 2025 20:00:27 +0530 Subject: [PATCH 85/98] fix size and temp dir path --- code/go/0chain.net/blobbercore/allocation/allocationchange.go | 2 +- code/go/0chain.net/blobbercore/filestore/storage.go | 2 +- code/go/0chain.net/blobbercore/handler/file_command_upload.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 5de70ec6b..d60534490 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -406,7 +406,7 @@ func deleteFromFileStore(allocationID string, deletedRefs []*reference.Ref, useR filestore.VERSION) if err != nil { logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), - zap.String("validation_root", res.LookupHash)) + zap.String("lookup_hash", res.LookupHash)) } }() diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 611157476..2ddaa5846 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -880,7 +880,7 @@ func (fs *FileStore) getPreCommitDir(allocationID string) string { func (fs *FileStore) getTempPathForFile(allocId, fileName, pathHash, connectionID string) string { fileName = sanitizeFileName(fileName) - return filepath.Join(fs.getAllocTempDir(allocId), fileName+"."+pathHash+"."+connectionID) + return filepath.Join(fs.getAllocTempDir(allocId), getPartialPath(pathHash, getDirLevelsForFiles())+"."+connectionID) } func (fs *FileStore) getPreCommitPathForFile(allocID, hash string, version int) string { diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index a59fc6ce5..d2859d69b 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -138,7 +138,6 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj result.Filename = cmd.fileChanger.Filename result.Size = fileOutputData.Size - allocationSize := allocation.GetConnectionObjSize(connectionID) + cmd.fileChanger.Size cmd.fileChanger.AllocationID = allocationObj.ID cmd.allocationChange = &allocation.AllocationChange{} @@ -176,6 +175,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj return result, err } } + allocationSize := allocation.GetConnectionObjSize(connectionID) if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize { return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") @@ -186,7 +186,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj // ProcessThumbnail flush thumbnail file to FileStorage if it has. func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocation) error { - logging.Logger.Info("ProcessThumbnail: ", zap.String("allocationID: ", cmd.fileChanger.AllocationID)) + connectionID := cmd.fileChanger.ConnectionID if cmd.thumbHeader != nil { defer cmd.thumbFile.Close() From ebc3c095719594ea0146c6685a27f6a1dc482d12 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Thu, 6 Feb 2025 18:01:55 +0530 Subject: [PATCH 86/98] dummy commit --- code/go/0chain.net/blobbercore/handler/handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index cc110b268..d0e65ea2e 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -285,7 +285,6 @@ func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF defer func() { tx.Rollback() }() - res, err := handler(ctx, r) return res, err } From 746b20855607a3f35ca8500819bc9c5ef11af1ce Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Fri, 14 Feb 2025 19:27:10 +0530 Subject: [PATCH 87/98] logging --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 56625bf28..94cfd701e 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -520,7 +520,7 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req clientSignV2 := ctx.Value(constants.ContextKeyClientSignatureHeaderV2Key).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, clientSignV2, publicKey) if !valid || err != nil { - return nil, common.NewError("invalid_signature", "could not verify the allocation owner") + return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+err.Error()) } var vm *writemarker.VersionMarker From 9af257f7661463dcb1c2ad40e11cdc5d288b8a0c Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Fri, 14 Feb 2025 20:54:54 +0530 Subject: [PATCH 88/98] log --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 94cfd701e..f5e201c53 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -520,6 +520,9 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req clientSignV2 := ctx.Value(constants.ContextKeyClientSignatureHeaderV2Key).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, clientSignV2, publicKey) if !valid || err != nil { + if !valid { + return nil, common.NewError("invalid_signature", "could not verify the allocation owner") + } return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+err.Error()) } From 8c8e45c2ed2272439e347187deaa98b42e6fef9f Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Thu, 20 Feb 2025 19:43:00 +0530 Subject: [PATCH 89/98] logging --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index f5e201c53..30ca46b33 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -520,10 +520,10 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req clientSignV2 := ctx.Value(constants.ContextKeyClientSignatureHeaderV2Key).(string) valid, err := verifySignatureFromRequest(allocationTx, clientSign, clientSignV2, publicKey) if !valid || err != nil { - if !valid { - return nil, common.NewError("invalid_signature", "could not verify the allocation owner") + if err != nil { + return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+err.Error()) } - return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+err.Error()) + return nil, common.NewError("invalid_signature", "could not verify the allocation owner") } var vm *writemarker.VersionMarker From b41ceaa9e16ad542240d8e6a4d66d13a6d6289a2 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Thu, 20 Feb 2025 21:18:52 +0530 Subject: [PATCH 90/98] logging --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 30ca46b33..8a55d85f0 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -989,7 +989,7 @@ func verifySignatureFromRequest(alloc, signV1, signV2, pbK string) (bool, error) hash = encryption.Hash(hashData) } if len(sign) < 64 { - return false, nil + return false, common.NewError("500", "len shorter than 64") } return encryption.Verify(pbK, sign, hash) } From 77a6fcb68a5f10d50868344363162101f283e598 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Mon, 24 Feb 2025 02:12:33 +0530 Subject: [PATCH 91/98] common sdk integrate --- code/go/0chain.net/blobber/logging.go | 2 +- code/go/0chain.net/blobber/settings.go | 20 +- code/go/0chain.net/blobber/zcn.go | 29 +- .../allocation/allocationchange.go | 2 +- .../allocation/copyfilechange_test.go | 8 +- .../0chain.net/blobbercore/allocation/dao.go | 2 +- .../allocation/deletefilechange_test.go | 6 +- .../allocation/file_changer_upload_test.go | 8 +- .../allocation/movefilechange_test.go | 6 +- .../blobbercore/allocation/multiop_test.go | 6 +- .../allocation/renamefilechange_test.go | 21 +- .../allocation/updatefilechange_test.go | 6 +- .../blobbercore/allocation/workers.go | 17 +- .../blobbercore/challenge/challenge.go | 17 +- .../blobbercore/challenge/protocol.go | 14 +- .../blobbercore/challenge/worker.go | 18 +- .../0chain.net/blobbercore/config/settings.go | 10 +- .../blobbercore/filestore/storage.go | 2 +- .../blobbercore/filestore/store_test.go | 2 +- .../blobbercore/filestore/tree_validation.go | 2 +- .../filestore/tree_validation_bench_test.go | 2 +- .../filestore/tree_validation_test.go | 2 +- .../blobbercore/handler/auth_ticket.go | 5 +- .../blobbercore/handler/chunk_encoder.go | 2 +- .../handler/file_command_delete.go | 2 +- .../handler/file_command_update.go | 4 +- .../handler/file_command_upload.go | 4 +- .../0chain.net/blobbercore/handler/handler.go | 6 +- .../blobbercore/handler/handler_common.go | 23 +- .../handler/handler_download_test.go | 17 +- .../handler/handler_objecttree_test.go | 17 +- .../handler/handler_refpath_test.go | 11 +- .../blobbercore/handler/handler_share_test.go | 13 +- .../blobbercore/handler/handler_test.go | 21 +- .../0chain.net/blobbercore/handler/health.go | 42 ++- .../handler/object_operation_handler.go | 2 +- .../object_operation_handler_bench_test.go | 2 +- .../handler/object_operation_handler_test.go | 43 +-- .../blobbercore/handler/protocol.go | 74 ++--- .../blobbercore/handler/storage_handler.go | 2 +- .../blobbercore/handler/tests_common_test.go | 18 +- .../0chain.net/blobbercore/handler/zcncore.go | 128 --------- code/go/0chain.net/blobbercore/mock/ctx.go | 2 +- code/go/0chain.net/blobbercore/mock/init.go | 6 +- .../blobbercore/readmarker/protocol.go | 33 +-- .../blobbercore/readmarker/readmarker.go | 2 +- code/go/0chain.net/blobbercore/util/json.go | 9 +- .../blobbercore/writemarker/mutex.go | 2 +- .../blobbercore/writemarker/protocol.go | 67 ++--- code/go/0chain.net/blobbercore/zcn/query.go | 25 +- code/go/0chain.net/core/chain/entity.go | 17 +- code/go/0chain.net/core/encryption/keys.go | 13 +- .../0chain.net/core/encryption/keys_test.go | 7 +- code/go/0chain.net/core/node/context.go | 2 +- code/go/0chain.net/core/node/self_node.go | 3 +- code/go/0chain.net/core/transaction/entity.go | 266 +----------------- code/go/0chain.net/core/transaction/http.go | 40 +-- code/go/0chain.net/core/transaction/nonce.go | 21 +- code/go/0chain.net/validator/main.go | 55 ++-- .../storage/challenge_handler.go | 30 +- .../validatorcore/storage/context.go | 2 +- .../storage/handler_integration_tests.go | 2 +- .../validatorcore/storage/models.go | 19 +- .../validatorcore/storage/models_test.go | 4 +- .../validatorcore/storage/protocol.go | 44 +-- .../storage/writemarker/entity_test.go | 2 +- go.mod | 23 +- go.sum | 66 +++-- 68 files changed, 460 insertions(+), 940 deletions(-) delete mode 100644 code/go/0chain.net/blobbercore/handler/zcncore.go diff --git a/code/go/0chain.net/blobber/logging.go b/code/go/0chain.net/blobber/logging.go index 339e8807c..98d38e38e 100644 --- a/code/go/0chain.net/blobber/logging.go +++ b/code/go/0chain.net/blobber/logging.go @@ -5,7 +5,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk_common/zcncore" ) func setupLogging() { diff --git a/code/go/0chain.net/blobber/settings.go b/code/go/0chain.net/blobber/settings.go index eb90322b2..72f631f16 100644 --- a/code/go/0chain.net/blobber/settings.go +++ b/code/go/0chain.net/blobber/settings.go @@ -9,7 +9,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk_common/core/transaction" ) type storageScCB struct { @@ -58,20 +58,24 @@ func (ssc *storageScCB) OnInfoAvailable(op int, status int, info string, errStr } func setStorageScConfigFromChain() error { - cb := &storageScCB{ - done: make(chan struct{}), + conf, err := transaction.GetConfig("storage_sc_config") + if err != nil { + return err } - err := zcncore.GetStorageSCConfig(cb) + + maxChallengeCompletionRoundsString := conf.Fields["max_challenge_completion_rounds"] + maxChallengeCompletionRoundsInt, err := strconv.ParseInt(maxChallengeCompletionRoundsString, 10, 64) if err != nil { return err } - <-cb.done - if cb.err != nil { + maxFileSizeString := conf.Fields["max_file_size"] + maxFileSizeInt64, err := strconv.ParseInt(maxFileSizeString, 10, 64) + if err != nil { return err } - config.StorageSCConfig.ChallengeCompletionTime = cb.cct - config.StorageSCConfig.MaxFileSize = cb.mfs + config.StorageSCConfig.ChallengeCompletionTime = maxChallengeCompletionRoundsInt + config.StorageSCConfig.MaxFileSize = maxFileSizeInt64 return nil } diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index 8417a81ea..87d95b979 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -2,6 +2,8 @@ package main import ( "fmt" + "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" @@ -10,10 +12,9 @@ import ( handleCommon "github.com/0chain/blobber/code/go/0chain.net/core/common/handler" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/gosdk/zboxcore/sdk" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/zcncore" "go.uber.org/zap" - "time" ) func registerOnChain() error { @@ -82,16 +83,28 @@ func setupServerChain() error { serverChain := chain.NewChainFromConfig() chain.SetServerChain(serverChain) - if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil { + //options := []int{ + // 0, + // 10, // MinConfirmation + // 20, // MinSubmit + // 3, // ConfirmationChainLength + // 3, // SharderConsensous + // 1, // QuerySleepTime + // 0, // VerifyOptimistic + //} + + err := client.InitSDK("{}", serverChain.BlockWorker, config.Configuration.ChainID, config.Configuration.SignatureScheme, 0, false) + if err != nil { return err } - if err := zcncore.SetWalletInfo(node.Self.GetWalletString(), false); err != nil { + + err = zcncore.SetGeneralWalletInfo(node.Self.GetWalletString(), config.Configuration.SignatureScheme) + if err != nil { return err } - if err := sdk.InitStorageSDK(node.Self.GetWalletString(), serverChain.BlockWorker, config.Configuration.ChainID, config.Configuration.SignatureScheme, - nil, 0); err != nil { - return err + if node.Self.GetWallet().IsSplit { + zcncore.RegisterZauthServer(serverChain.ZauthServer) } fmt.Print(" [OK]\n") diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index d60534490..b1395caa4 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -13,7 +13,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "golang.org/x/sync/errgroup" "go.uber.org/zap" diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go index f0ab47414..121cad230 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go @@ -15,9 +15,9 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/constants" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" mocket "github.com/selvatico/go-mocket" "go.uber.org/zap" "google.golang.org/grpc/metadata" @@ -38,7 +38,7 @@ func TestBlobberCore_CopyFile(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerID = client.Id() testCases := []struct { name string diff --git a/code/go/0chain.net/blobbercore/allocation/dao.go b/code/go/0chain.net/blobbercore/allocation/dao.go index a62005e4f..6f67b1943 100644 --- a/code/go/0chain.net/blobbercore/allocation/dao.go +++ b/code/go/0chain.net/blobbercore/allocation/dao.go @@ -5,7 +5,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/errors" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "gorm.io/gorm" ) diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go index b94dc9d7b..7d8ba36f8 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go @@ -13,8 +13,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" mocket "github.com/selvatico/go-mocket" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -32,7 +32,7 @@ func TestBlobberCore_DeleteFile(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerID = client.Id() testCases := []struct { name string context metadata.MD diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go index dced40d75..f1f8964c1 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go @@ -17,9 +17,9 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/constants" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" mocket "github.com/selvatico/go-mocket" "go.uber.org/zap" "google.golang.org/grpc/metadata" @@ -39,7 +39,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerID = client.Id() testCases := []struct { name string diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go index befa83566..0e89ec582 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go @@ -16,8 +16,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" "go.uber.org/zap" "google.golang.org/grpc/metadata" ) @@ -37,7 +37,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerID = client.Id() testCases := []struct { name string diff --git a/code/go/0chain.net/blobbercore/allocation/multiop_test.go b/code/go/0chain.net/blobbercore/allocation/multiop_test.go index 917cd9030..8cde60fc5 100644 --- a/code/go/0chain.net/blobbercore/allocation/multiop_test.go +++ b/code/go/0chain.net/blobbercore/allocation/multiop_test.go @@ -13,8 +13,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" mocket "github.com/selvatico/go-mocket" "github.com/stretchr/testify/require" ) @@ -30,7 +30,7 @@ func TestMultiOp(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerID = client.Id() datastore.MocketTheStore(t, true) ctx := datastore.GetStore().CreateTransaction(context.TODO()) setupDbMock() diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go index c895c7003..66b44da29 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -17,10 +17,11 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/config" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" - zencryption "github.com/0chain/gosdk/zboxcore/encryption" - "github.com/0chain/gosdk/zcncore" + coreNetwork "github.com/0chain/gosdk/core/conf" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" + zencryption "github.com/0chain/gosdk_common/zboxcore/encryption" + "github.com/0chain/gosdk_common/zcncore" "github.com/DATA-DOG/go-sqlmock" mocket "github.com/selvatico/go-mocket" "github.com/stretchr/testify/require" @@ -55,7 +56,7 @@ func setup(t *testing.T) { if err != nil { t.Fatal(err) } - if err := zcncore.SetWalletInfo(string(wBlob), true); err != nil { + if err := zcncore.SetWalletInfo(string(wBlob), "bls0chain", true); err != nil { t.Fatal(err) } @@ -66,10 +67,10 @@ func setup(t *testing.T) { }, ), ) - server := httptest.NewServer( + _ = httptest.NewServer( http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - n := zcncore.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}} + n := coreNetwork.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}} blob, err := json.Marshal(n) if err != nil { t.Fatal(err) @@ -81,10 +82,6 @@ func setup(t *testing.T) { }, ), ) - - if err := zcncore.InitZCNSDK(server.URL, "ed25519"); err != nil { - t.Fatal(err) - } } func setupMockForFileManagerInit(mock sqlmock.Sqlmock) { mock.ExpectBegin() @@ -151,7 +148,7 @@ func TestBlobberCore_RenameFile(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerID = client.Id() testCases := []struct { name string diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index 5b4f4e162..07c789d78 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -12,8 +12,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" mocket "github.com/selvatico/go-mocket" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,7 +36,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = client.GetClientID() + alloc.OwnerID = client.Id() testCases := []struct { name string diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go index b5475b81d..97f916630 100644 --- a/code/go/0chain.net/blobbercore/allocation/workers.go +++ b/code/go/0chain.net/blobbercore/allocation/workers.go @@ -12,6 +12,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + coreTxn "github.com/0chain/gosdk_common/core/transaction" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" @@ -306,20 +308,15 @@ type finalizeRequest struct { } func sendFinalizeAllocation(allocationID string) { - var tx, err = transaction.NewTransactionEntity() - if err != nil { - logging.Logger.Error("creating new transaction entity", zap.Error(err)) - return - } - var request finalizeRequest request.AllocationID = allocationID - err = tx.ExecuteSmartContract( + _, _, _, _, err := coreTxn.SmartContractTxn( transaction.STORAGE_CONTRACT_ADDRESS, - transaction.FINALIZE_ALLOCATION, - request, - 0) + coreTxn.SmartContractTxnData{ + Name: transaction.FINALIZE_ALLOCATION, + InputArgs: request, + }, true) if err != nil { logging.Logger.Error("sending finalize allocation", zap.Error(err)) return diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index e8f37cee7..5fb55d152 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -9,6 +9,7 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/core/node" + coreTxn "github.com/0chain/gosdk_common/core/transaction" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -162,7 +163,7 @@ func validateOnValidators(ctx context.Context, c *ChallengeEntity) error { return nil } -func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*transaction.Transaction, error) { +func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*coreTxn.Transaction, error) { createdTime := common.ToTime(c.CreatedAt) logging.Logger.Info("[challenge]verify: ", @@ -186,13 +187,6 @@ func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*transactio return nil, nil } - txn, err := transaction.NewTransactionEntity() - if err != nil { - logging.Logger.Error("[challenge]createTxn", zap.Error(err)) - c.CancelChallenge(ctx, err) - return nil, nil - } - sn := &ChallengeResponse{} sn.ChallengeID = c.ChallengeID for _, vt := range c.ValidationTickets { @@ -201,14 +195,17 @@ func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*transactio } } - err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.CHALLENGE_RESPONSE, sn, 0) + _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{ + Name: transaction.CHALLENGE_RESPONSE, + InputArgs: sn, + }, false) if err != nil { logging.Logger.Info("Failed submitting challenge to the mining network", zap.String("err:", err.Error())) c.CancelChallenge(ctx, err) return nil, nil } - err = UpdateChallengeTimingTxnSubmission(c.ChallengeID, txn.CreationDate) + err = UpdateChallengeTimingTxnSubmission(c.ChallengeID, common.Timestamp(txn.CreationDate)) if err != nil { logging.Logger.Error("[challengetiming]txn_submission", zap.Any("challenge_id", c.ChallengeID), diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 65c0c84f6..8d669d2e9 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -14,13 +14,13 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "github.com/0chain/blobber/code/go/0chain.net/core/util" - sdkUtil "github.com/0chain/gosdk/core/util" + coretxn "github.com/0chain/gosdk_common/core/transaction" + sdkUtil "github.com/0chain/gosdk_common/core/util" "github.com/remeh/sizedwaitgroup" "gorm.io/gorm" @@ -347,11 +347,11 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { return nil } -func (cr *ChallengeEntity) VerifyChallengeTransaction(ctx context.Context, txn *transaction.Transaction) error { +func (cr *ChallengeEntity) VerifyChallengeTransaction(ctx context.Context, txn *coretxn.Transaction) error { if len(cr.LastCommitTxnIDs) > 0 { for _, lastTxn := range cr.LastCommitTxnIDs { logging.Logger.Info("[challenge]commit: Verifying the transaction : " + lastTxn) - t, err := transaction.VerifyTransaction(lastTxn, chain.GetServerChain()) + t, err := coretxn.VerifyTransaction(lastTxn) if err == nil { cr.SaveChallengeResult(ctx, t, false) return nil @@ -362,11 +362,11 @@ func (cr *ChallengeEntity) VerifyChallengeTransaction(ctx context.Context, txn * logging.Logger.Info("Verifying challenge response to blockchain.", zap.String("txn", txn.Hash), zap.String("challenge_id", cr.ChallengeID)) var ( - t *transaction.Transaction + t *coretxn.Transaction err error ) for i := 0; i < 3; i++ { - t, err = transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce()) + t, err = coretxn.VerifyTransaction(txn.Hash) if err == nil { break } @@ -403,7 +403,7 @@ func IsEntityNotFoundError(err error) bool { return strings.Contains(err.Error(), EntityNotFound) } -func (cr *ChallengeEntity) SaveChallengeResult(ctx context.Context, t *transaction.Transaction, toAdd bool) { +func (cr *ChallengeEntity) SaveChallengeResult(ctx context.Context, t *coretxn.Transaction, toAdd bool) { cr.statusMutex.Lock() cr.Status = Committed cr.statusMutex.Unlock() diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index b4ef094d0..c4fb76c7c 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -2,15 +2,15 @@ package challenge import ( "context" + "encoding/json" "sync" "time" - "github.com/0chain/gosdk/zcncore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "github.com/0chain/gosdk_common/core/screstapi" + coreTxn "github.com/0chain/gosdk_common/core/transaction" "github.com/emirpasic/gods/maps/treemap" "go.uber.org/zap" "golang.org/x/sync/semaphore" @@ -95,7 +95,15 @@ func getRoundWorker(ctx context.Context) { } func setRound() { - currentRound, _ := zcncore.GetRoundFromSharders() + res, err := screstapi.MakeSCRestAPICall("", "/v1/current-round", nil, "") + if err != nil { + logging.Logger.Error("getRoundWorker", zap.Error(err)) + } + var currentRound int64 + err = json.Unmarshal(res, ¤tRound) + if err != nil { + logging.Logger.Error("getRoundWorker", zap.Error(err)) + } if roundInfo.LastRoundDiff == 0 { roundInfo.LastRoundDiff = 1000 @@ -183,7 +191,7 @@ func commitOnChainWorker(ctx context.Context) { for _, challenge := range challenges { chall := challenge var ( - txn *transaction.Transaction + txn *coreTxn.Transaction err error ) _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { diff --git a/code/go/0chain.net/blobbercore/config/settings.go b/code/go/0chain.net/blobbercore/config/settings.go index f12f26cca..7fd0e9e30 100644 --- a/code/go/0chain.net/blobbercore/config/settings.go +++ b/code/go/0chain.net/blobbercore/config/settings.go @@ -8,8 +8,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/errors" - "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk/zboxcore/sdk" + "github.com/0chain/gosdk_common/constants" "go.uber.org/zap" "gorm.io/gorm" ) @@ -102,7 +102,7 @@ func Update(ctx context.Context, db *gorm.DB) error { } // ReloadFromChain load and refresh latest settings from blockchain -func ReloadFromChain(ctx context.Context, db *gorm.DB) (*zcncore.Blobber, error) { +func ReloadFromChain(ctx context.Context, db *gorm.DB) (*sdk.Blobber, error) { if db == nil { return nil, errors.Throw(constants.ErrInvalidParameter, "db") } @@ -115,7 +115,7 @@ func ReloadFromChain(ctx context.Context, db *gorm.DB) (*zcncore.Blobber, error) } Configuration.Capacity = int64(b.Capacity) - Configuration.NumDelegates = *b.StakePoolSettings.NumDelegates + Configuration.NumDelegates = b.StakePoolSettings.NumDelegates if token, err := b.Terms.ReadPrice.ToToken(); err != nil { return nil, err @@ -129,6 +129,6 @@ func ReloadFromChain(ctx context.Context, db *gorm.DB) (*zcncore.Blobber, error) Configuration.WritePrice = token } - Configuration.ServiceCharge = *b.StakePoolSettings.ServiceCharge + Configuration.ServiceCharge = b.StakePoolSettings.ServiceCharge return b, Update(ctx, db) } diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 2ddaa5846..083f3d3e3 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -45,7 +45,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/util" + "github.com/0chain/gosdk_common/core/util" "go.uber.org/zap" "golang.org/x/crypto/sha3" "golang.org/x/sys/unix" diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go index 66ad81b25..21e4da514 100644 --- a/code/go/0chain.net/blobbercore/filestore/store_test.go +++ b/code/go/0chain.net/blobbercore/filestore/store_test.go @@ -21,7 +21,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/seqpriorityqueue" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/util" + "github.com/0chain/gosdk_common/core/util" "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/require" "go.uber.org/zap" diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 1ceffce1e..b56a742ff 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -17,7 +17,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/seqpriorityqueue" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/util" + "github.com/0chain/gosdk_common/core/util" "github.com/minio/sha256-simd" "go.uber.org/zap" ) diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go index 4d12fcfff..17c218d0b 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go @@ -7,7 +7,7 @@ import ( "os" "testing" - "github.com/0chain/gosdk/core/util" + "github.com/0chain/gosdk_common/core/util" "github.com/minio/sha256-simd" ) diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go index 5f10ac11c..7f16119e5 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/0chain/gosdk/core/util" + "github.com/0chain/gosdk_common/core/util" "github.com/stretchr/testify/require" ) diff --git a/code/go/0chain.net/blobbercore/handler/auth_ticket.go b/code/go/0chain.net/blobbercore/handler/auth_ticket.go index 48f973252..faa119c08 100644 --- a/code/go/0chain.net/blobbercore/handler/auth_ticket.go +++ b/code/go/0chain.net/blobbercore/handler/auth_ticket.go @@ -3,12 +3,13 @@ package handler import ( "context" "fmt" + "net/http" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/common/core/common" - "github.com/0chain/gosdk/core/encryption" + "github.com/0chain/gosdk_common/core/encryption" "go.uber.org/zap" - "net/http" ) // swagger:model AuthTicketResponse diff --git a/code/go/0chain.net/blobbercore/handler/chunk_encoder.go b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go index 6665b9dac..97e0d8f7d 100644 --- a/code/go/0chain.net/blobbercore/handler/chunk_encoder.go +++ b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go @@ -4,7 +4,7 @@ import ( "bytes" "errors" - zencryption "github.com/0chain/gosdk/zboxcore/encryption" + zencryption "github.com/0chain/gosdk_common/zboxcore/encryption" ) // ChunkEncoder encode/decode chunk data diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index aef760aaf..2fcdebc96 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -6,7 +6,7 @@ import ( "net/http" "path/filepath" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index aaadff53d..1bfe4df13 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -14,8 +14,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - sdkConst "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/zboxcore/fileref" + sdkConst "github.com/0chain/gosdk_common/constants" + "github.com/0chain/gosdk_common/zboxcore/fileref" ) const ( diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index d2859d69b..4dbd00544 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -18,8 +18,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk_common/constants" + "github.com/0chain/gosdk_common/zboxcore/fileref" ) const ( diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index d0e65ea2e..eeb943e8a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -14,14 +14,14 @@ import ( "strings" "time" - "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk_common/core/zcncrypto" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "github.com/go-openapi/runtime/middleware" - "github.com/0chain/gosdk/constants" - "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk_common/constants" + "github.com/0chain/gosdk_common/zboxcore/fileref" "github.com/didip/tollbooth/v6/limiter" "github.com/gorilla/mux" "github.com/spf13/viper" diff --git a/code/go/0chain.net/blobbercore/handler/handler_common.go b/code/go/0chain.net/blobbercore/handler/handler_common.go index 8c9a46bbd..f9e2c41a2 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_common.go +++ b/code/go/0chain.net/blobbercore/handler/handler_common.go @@ -13,7 +13,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk/core/client" "go.uber.org/zap" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" @@ -77,16 +77,19 @@ func HomepageHandler(w http.ResponseWriter, r *http.Request) { ) fmt.Fprintf(w, "
Miners ...\n") - network := zcncore.GetNetwork() - for _, miner := range network.Miners { - fmt.Fprintf(w, "%v\n", miner) - } - fmt.Fprintf(w, "
\n") - fmt.Fprintf(w, "
Sharders ...\n") - for _, sharder := range network.Sharders { - fmt.Fprintf(w, "%v\n", sharder) + network, err := client.GetNetwork(context.Background()) + if err == nil { + fmt.Fprintf(w, "
Miners ...\n") + for _, miner := range network.Miners { + fmt.Fprintf(w, "%v\n", miner) + } + fmt.Fprintf(w, "
\n") + fmt.Fprintf(w, "
Sharders ...\n") + for _, sharder := range network.Sharders { + fmt.Fprintf(w, "%v\n", sharder) + } + fmt.Fprintf(w, "
\n") } - fmt.Fprintf(w, "
\n") fmt.Fprintf(w, "
") fmt.Fprintf(w, "
Running since %v (Total elapsed time: %v)
\n", StartTime.Format(common.DateTimeFormat), time.Since(StartTime)) fmt.Fprintf(w, "
") diff --git a/code/go/0chain.net/blobbercore/handler/handler_download_test.go b/code/go/0chain.net/blobbercore/handler/handler_download_test.go index 70a0e5dfe..e02abab31 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_download_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_download_test.go @@ -14,11 +14,11 @@ import ( "testing" "time" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" - zencryption "github.com/0chain/gosdk/zboxcore/encryption" - "github.com/0chain/gosdk/zboxcore/fileref" - "github.com/0chain/gosdk/zboxcore/zboxutil" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" + zencryption "github.com/0chain/gosdk_common/zboxcore/encryption" + "github.com/0chain/gosdk_common/zboxcore/fileref" + "github.com/0chain/gosdk_common/zboxcore/zboxutil" "github.com/DATA-DOG/go-sqlmock" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" @@ -60,11 +60,8 @@ func TestHandlers_Download(t *testing.T) { clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}` guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}` - require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) - clients := client.GetClients() - - ownerClient, guestClient := clients[0], clients[1] - + ownerClient, _ := client.PopulateClient(clientJson, "bls0chain") + guestClient, _ := client.PopulateClient(guestClientJson, "bls0chain") ownerScheme, err := getEncryptionScheme(ownerClient.Mnemonic) if err != nil { t.Fatal(err) diff --git a/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go b/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go index 79ea68676..3217712c5 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go @@ -4,6 +4,7 @@ package handler import ( + "encoding/json" "net/http" "net/http/httptest" "os" @@ -11,8 +12,8 @@ import ( "testing" "time" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" "github.com/DATA-DOG/go-sqlmock" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" @@ -42,18 +43,18 @@ func TestHandlers_ObjectTree(t *testing.T) { setup(t) clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}` - guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}` - require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) - clients := client.GetClients() - - ownerClient := clients[0] + ownerClient := zcncrypto.Wallet{} + err := json.Unmarshal([]byte(clientJson), &ownerClient) + require.NoError(t, err) + client.SetWallet(ownerClient) + client.SetSignatureScheme("bls0chain") router, handlers := setupObjectTreeHandlers() sch := zcncrypto.NewSignatureScheme("bls0chain") //sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" - _, err := sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") + _, err = sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe") if err != nil { t.Fatal(err) } diff --git a/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go b/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go index 04b47d2a5..5926bfcdf 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go @@ -13,12 +13,11 @@ import ( "testing" "time" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" "github.com/DATA-DOG/go-sqlmock" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" @@ -44,12 +43,8 @@ func TestHandlers_ReferencePath(t *testing.T) { setup(t) clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}` - guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}` - require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) - clients := client.GetClients() - - ownerClient := clients[0] + ownerClient, _ := client.PopulateClient(clientJson, "bls0chain") router, handlers := setupReferencePathHandlers() diff --git a/code/go/0chain.net/blobbercore/handler/handler_share_test.go b/code/go/0chain.net/blobbercore/handler/handler_share_test.go index 9a288997a..cb5fe690d 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_share_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_share_test.go @@ -15,9 +15,9 @@ import ( "testing" "time" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" - "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" + "github.com/0chain/gosdk_common/zboxcore/fileref" "github.com/DATA-DOG/go-sqlmock" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" @@ -61,13 +61,8 @@ func TestHandlers_Share(t *testing.T) { setup(t) clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}` - guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}` - - require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) - clients := client.GetClients() - - ownerClient := clients[0] + ownerClient, _ := client.PopulateClient(clientJson, "bls0chain") router, handlers := setupShareHandlers() sch := zcncrypto.NewSignatureScheme("bls0chain") diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index a7ed52633..ffba186ce 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -18,13 +18,12 @@ import ( "github.com/DATA-DOG/go-sqlmock" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zboxcore/client" - "github.com/0chain/gosdk/zboxcore/fileref" - "github.com/0chain/gosdk/zboxcore/marker" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" + "github.com/0chain/gosdk_common/zboxcore/fileref" + "github.com/0chain/gosdk_common/zboxcore/marker" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" @@ -49,7 +48,7 @@ func resetMockFileBlock() { mockFileBlock = []byte("mock") } -func signHash(client *client.Client, hash string) (string, error) { +func signHash(client zcncrypto.Wallet, hash string) (string, error) { retSignature := "" for _, kv := range client.Keys { ss := zcncrypto.NewSignatureScheme("bls0chain") @@ -155,7 +154,7 @@ func isEndpointAllowGetReq(name string) bool { } } -func GetAuthTicketForEncryptedFile(ownerClient *client.Client, allocationID, remotePath, fileHash, clientID, encPublicKey string) (string, error) { +func GetAuthTicketForEncryptedFile(ownerClient zcncrypto.Wallet, allocationID, remotePath, fileHash, clientID, encPublicKey string) (string, error) { at := &marker.AuthTicket{} at.AllocationID = allocationID at.OwnerID = ownerClient.ClientID @@ -190,13 +189,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { setup(t) clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}` - guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}` - - require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain")) - clients := client.GetClients() - - ownerClient := clients[0] - + ownerClient, _ := client.PopulateClient(clientJson, "bls0chain") router, handlers := setupTestHandlers() sch := zcncrypto.NewSignatureScheme("bls0chain") diff --git a/code/go/0chain.net/blobbercore/handler/health.go b/code/go/0chain.net/blobbercore/handler/health.go index e4da97bf7..b0e406c44 100644 --- a/code/go/0chain.net/blobbercore/handler/health.go +++ b/code/go/0chain.net/blobbercore/handler/health.go @@ -7,6 +7,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + coreTxn "github.com/0chain/gosdk_common/core/transaction" "go.uber.org/zap" ) @@ -29,48 +30,41 @@ func getBlobberHealthCheckError() error { return err } -func BlobberHealthCheck() (*transaction.Transaction, error) { +func BlobberHealthCheck() (string, error) { if config.Configuration.Capacity == 0 { setBlobberHealthCheckError(ErrBlobberHasRemoved) - return nil, ErrBlobberHasRemoved + return "", ErrBlobberHasRemoved } - txn, err := transaction.NewTransactionEntity() - if err != nil { - setBlobberHealthCheckError(err) - return nil, err - } - - err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, - transaction.BLOBBER_HEALTH_CHECK, common.Now(), 0) - if err != nil { + _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{ + Name: transaction.BLOBBER_HEALTH_CHECK, + InputArgs: common.Now(), + }, true) + if err != nil || txn == nil { logging.Logger.Error("Failed to health check blobber on the blockchain", zap.Error(err)) setBlobberHealthCheckError(err) - return nil, err + return "", err } setBlobberHealthCheckError(nil) - return txn, nil + return txn.Hash, nil } -func ValidatorHealthCheck() (*transaction.Transaction, error) { - - txn, err := transaction.NewTransactionEntity() - - if err != nil { - - return nil, err - } +func ValidatorHealthCheck() (string, error) { + _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{ + Name: transaction.VALIDATOR_HEALTH_CHECK, + InputArgs: common.Now(), + }, true) - if err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.VALIDATOR_HEALTH_CHECK, common.Now(), 0); err != nil { + if err != nil || txn == nil { logging.Logger.Error("Failed to health check validator on the blockchain", zap.Error(err)) - return nil, err + return "", err } - return txn, err + return txn.Hash, err } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 79b254832..b73e45ea4 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -16,7 +16,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go index adcb74109..0d0094022 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go @@ -11,7 +11,7 @@ package handler // "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" // "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" // "github.com/0chain/blobber/code/go/0chain.net/blobbercore/mock" -// "github.com/0chain/gosdk/zboxcore/sdk" +// "github.com/0chain/gosdk_common/zboxcore/sdk" // ) // func BenchmarkUploadFileWithDisk(b *testing.B) { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index ccc627dc9..b1f33365b 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -16,23 +16,24 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/zcncrypto" mocket "github.com/selvatico/go-mocket" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk_common/zboxcore/fileref" - "github.com/0chain/gosdk/zboxcore/blockchain" + "github.com/0chain/gosdk_common/zboxcore/blockchain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - c_common "github.com/0chain/gosdk/core/common" - "github.com/0chain/gosdk/zboxcore/marker" + c_common "github.com/0chain/gosdk_common/core/common" + "github.com/0chain/gosdk_common/zboxcore/marker" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "github.com/stretchr/testify/require" "testing" @@ -56,10 +57,8 @@ func TestDownloadFile(t *testing.T) { ) ts := time.Now().Add(time.Hour) var mockLongTimeInFuture = common.Timestamp(ts.Unix()) + common.Timestamp(time.Second*1000) - var mockClient client.Client - require.NoError(t, json.Unmarshal([]byte(mockClientWallet), &mockClient)) - var mockOwner client.Client - require.NoError(t, json.Unmarshal([]byte(mockOwnerWallet), &mockOwner)) + mockClient, _ := client.PopulateClient(mockClientWallet, "bls0chain") + mockOwner, _ := client.PopulateClient(mockOwnerWallet, "bls0chain") var ( now = c_common.Timestamp(time.Now().Unix()) ) @@ -84,7 +83,7 @@ func TestDownloadFile(t *testing.T) { isRevoked bool isFundedBlobber bool isFunded0Chain bool - payerId client.Client + payerId zcncrypto.Wallet // client input from gosdk's BlockDownloadRequest, inData blockDownloadRequest @@ -121,15 +120,17 @@ func TestDownloadFile(t *testing.T) { if p.useAuthTicket { authTicket := &marker.AuthTicket{ AllocationID: p.inData.allocationID, - ClientID: client.GetClientID(), + ClientID: client.Wallet().ClientID, Expiration: int64(time.Duration(now) + 10000*time.Second), OwnerID: mockOwner.ClientID, Timestamp: int64(common.Now()), FilePathHash: p.inData.pathHash, } - require.NoError(t, client.PopulateClient(mockOwnerWallet, "bls0chain")) + _, err := client.PopulateClient(mockOwnerWallet, "bls0chain") + require.NoError(t, err) require.NoError(t, authTicket.Sign()) - require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain")) + _, err = client.PopulateClient(mockClientWallet, "bls0chain") + require.NoError(t, err) authTicketBytes, _ := json.Marshal(authTicket) auth := base64.StdEncoding.EncodeToString(authTicketBytes) req.Header.Set("X-Auth-Token", auth) @@ -139,8 +140,8 @@ func TestDownloadFile(t *testing.T) { } } - makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string) ([]byte, error) { - return func(scAddress string, relativePath string, params map[string]string) ([]byte, error) { + makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string, options ...string) ([]byte, error) { + return func(scAddress string, relativePath string, params map[string]string, options ...string) ([]byte, error) { require.New(t) require.EqualValues(t, scAddress, transaction.STORAGE_CONTRACT_ADDRESS) switch relativePath { @@ -290,9 +291,9 @@ func TestDownloadFile(t *testing.T) { setupCtx := func(p parameters) context.Context { ctx := context.TODO() - ctx = context.WithValue(ctx, constants.ContextKeyClient, client.GetClientID()) + ctx = context.WithValue(ctx, constants.ContextKeyClient, client.Id()) ctx = context.WithValue(ctx, constants.ContextKeyAllocation, p.inData.allocationTx) - ctx = context.WithValue(ctx, constants.ContextKeyClientKey, client.GetClientPublicKey()) + ctx = context.WithValue(ctx, constants.ContextKeyClientKey, client.PublicKey()) ctx = datastore.GetStore().CreateTransaction(ctx) @@ -455,9 +456,11 @@ func TestDownloadFile(t *testing.T) { func(t *testing.T) { setupParams(&test.parameters) if test.parameters.isOwner { - require.NoError(t, client.PopulateClient(mockOwnerWallet, "bls0chain")) + _, err := client.PopulateClient(mockOwnerWallet, "bls0chain") + require.NoError(t, err) } else { - require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain")) + _, err := client.PopulateClient(mockClientWallet, "bls0chain") + require.NoError(t, err) } transaction.MakeSCRestAPICall = makeMockMakeSCRestAPICall(t, test.parameters) request := setupRequest(test.parameters) diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go index 8a5993e8b..9a2141db6 100644 --- a/code/go/0chain.net/blobbercore/handler/protocol.go +++ b/code/go/0chain.net/blobbercore/handler/protocol.go @@ -3,19 +3,19 @@ package handler import ( "context" "errors" - "fmt" "sync" - "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/gosdk_common/core/client" + coreTxn "github.com/0chain/gosdk_common/core/transaction" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - "github.com/0chain/blobber/code/go/0chain.net/core/util" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk_common/zcncore" "go.uber.org/zap" ) @@ -80,19 +80,13 @@ func RegisterBlobber(ctx context.Context) error { }) if err != nil { // blobber is not registered yet - txn, err := sendSmartContractBlobberAdd(ctx) - if err != nil { - logging.Logger.Error("Error when sending add request to blockchain", zap.Any("err", err)) - return err - } - - t, err := TransactionVerify(txn) + txn, err := sendSmartContractBlobberAdd() if err != nil { - logging.Logger.Error("Failed to verify blobber register transaction", zap.Any("err", err), zap.String("txn.Hash", txn.Hash)) + logging.Logger.Error("Error in add blobber", zap.Any("err", err)) return err } - logging.Logger.Info("Verified blobber register transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) + logging.Logger.Info("Verified blobber register transaction", zap.String("txn_hash", txn.Hash), zap.Any("txn_output", txn.TransactionOutput)) return nil } @@ -106,38 +100,31 @@ func RegisterBlobber(ctx context.Context) error { } func RefreshPriceOnChain(ctx context.Context) error { - txn, err := sendSmartContractBlobberAdd(ctx) + txn, err := sendSmartContractBlobberAdd() if err != nil { - return err - } - - if t, err := TransactionVerify(txn); err != nil { logging.Logger.Error("Failed to verify price refresh transaction", zap.Any("err", err), zap.String("txn.Hash", txn.Hash)) - } else { - logging.Logger.Info("Verified price refresh transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) + return err } - return err + logging.Logger.Info("Verified price refresh transaction", zap.String("txn_hash", txn.Hash), zap.Any("txn_output", txn.TransactionOutput)) + return nil } // sendSmartContractBlobberAdd Add or update blobber on blockchain -func sendSmartContractBlobberAdd(ctx context.Context) (*transaction.Transaction, error) { +func sendSmartContractBlobberAdd() (*coreTxn.Transaction, error) { // initialize storage node (ie blobber) - txn, err := transaction.NewTransactionEntity() - if err != nil { - return nil, err - } - sn, err := getStorageNode() if err != nil { return nil, err } - err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, - transaction.ADD_BLOBBER_SC_NAME, sn, 0) + _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{ + Name: transaction.ADD_BLOBBER_SC_NAME, + InputArgs: sn, + }, true) if err != nil { logging.Logger.Error("Failed to set blobber on the blockchain", - zap.String("err:", err.Error())) + zap.String("err:", err.Error()), zap.Any("Txn", txn), zap.Any("ClientFee", client.TxnFee())) return nil, err } @@ -162,39 +149,20 @@ var ErrValidatorHasRemoved = errors.New("validator has been removed") // ErrValidatorNotFound it is not registered on chain var ErrValidatorNotFound = errors.New("validator is not found") -func TransactionVerify(txn *transaction.Transaction) (t *transaction.Transaction, err error) { - msg := fmt.Sprintf("Verifying transaction: max_retries: %d", util.MAX_RETRIES) - logging.Logger.Info(msg) - for i := 0; i < util.MAX_RETRIES; i++ { - time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second) - if t, err = transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce()); err == nil { - return t, nil - } - } - - return nil, errors.New("[txn]max retries exceeded with " + txn.Hash) -} - // SendHealthCheck send heartbeat to blockchain func SendHealthCheck(provider common.ProviderType) (string, error) { - var txn *transaction.Transaction + var hash string var err error switch provider { case common.ProviderTypeBlobber: - txn, err = BlobberHealthCheck() + hash, err = BlobberHealthCheck() case common.ProviderTypeValidator: - txn, err = ValidatorHealthCheck() + hash, err = ValidatorHealthCheck() default: return "", errors.New("unknown provider type") } - if err != nil { - return "", err - } - - _, err = TransactionVerify(txn) - - return txn.Hash, err + return hash, err } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 8a55d85f0..455d29519 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -12,7 +12,7 @@ import ( "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" diff --git a/code/go/0chain.net/blobbercore/handler/tests_common_test.go b/code/go/0chain.net/blobbercore/handler/tests_common_test.go index 6ea548768..461a042e9 100644 --- a/code/go/0chain.net/blobbercore/handler/tests_common_test.go +++ b/code/go/0chain.net/blobbercore/handler/tests_common_test.go @@ -15,9 +15,11 @@ import ( "strings" "testing" + coreNetwork "github.com/0chain/gosdk_common/core/conf" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk_common/core/zcncrypto" + "github.com/0chain/gosdk_common/zcncore" ) func setup(t *testing.T) { @@ -30,7 +32,7 @@ func setup(t *testing.T) { if err != nil { t.Fatal(err) } - if err := zcncore.SetWalletInfo(string(wBlob), true); err != nil { + if err := zcncore.SetWalletInfo(string(wBlob), "bls0chain", true); err != nil { t.Fatal(err) } @@ -41,10 +43,10 @@ func setup(t *testing.T) { }, ), ) - server := httptest.NewServer( + _ = httptest.NewServer( http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - n := zcncore.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}} + n := coreNetwork.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}} blob, err := json.Marshal(n) if err != nil { t.Fatal(err) @@ -57,9 +59,9 @@ func setup(t *testing.T) { ), ) - if err := zcncore.InitZCNSDK(server.URL, "ed25519"); err != nil { - t.Fatal(err) - } + // if err := zcncore.InitZCNSDK(server.URL, "ed25519"); err != nil { + // t.Fatal(err) + // } } type MockFileStore struct { diff --git a/code/go/0chain.net/blobbercore/handler/zcncore.go b/code/go/0chain.net/blobbercore/handler/zcncore.go deleted file mode 100644 index 36df8290a..000000000 --- a/code/go/0chain.net/blobbercore/handler/zcncore.go +++ /dev/null @@ -1,128 +0,0 @@ -package handler - -import ( - "sync" - - "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/zcncore" -) - -type ZCNStatus struct { - wg *sync.WaitGroup - success bool - balance int64 - info string -} - -func (zcn *ZCNStatus) OnBalanceAvailable(status int, value int64, info string) { - defer zcn.wg.Done() - if status == zcncore.StatusSuccess { - zcn.success = true - } else { - zcn.success = false - } - zcn.balance = value -} - -func (zcn *ZCNStatus) OnInfoAvailable(op, status int, info, err string) { - defer zcn.wg.Done() - if status == zcncore.StatusSuccess { - zcn.success = true - } else { - zcn.success = false - } - zcn.info = info -} - -func (zcn *ZCNStatus) OnTransactionComplete(t *zcncore.Transaction, status int) { - defer zcn.wg.Done() - if status == zcncore.StatusSuccess { - zcn.success = true - } else { - zcn.success = false - } -} - -func (zcn *ZCNStatus) OnVerifyComplete(t *zcncore.Transaction, status int) { - defer zcn.wg.Done() - if status == zcncore.StatusSuccess { - zcn.success = true - } else { - zcn.success = false - } -} - -func (zcn *ZCNStatus) OnAuthComplete(t *zcncore.Transaction, status int) {} - -func CheckBalance() (float64, error) { - wg := &sync.WaitGroup{} - statusBar := &ZCNStatus{wg: wg} - wg.Add(1) - err := zcncore.GetBalance(statusBar) - if err != nil { - return 0, common.NewError("check_balance_failed", "Call to GetBalance failed with err: "+err.Error()) - } - wg.Wait() - if !statusBar.success { - return 0, nil - } - return zcncore.ConvertToToken(statusBar.balance), nil -} - -func CallFaucet() error { - wg := &sync.WaitGroup{} - statusBar := &ZCNStatus{wg: wg} - txn, err := zcncore.NewTransaction(statusBar, 0, 0) - if err != nil { - return common.NewError("call_faucet_failed", "Failed to create new transaction with err: "+err.Error()) - } - wg.Add(1) - _, err = txn.ExecuteSmartContract(zcncore.FaucetSmartContractAddress, "pour", "Blobber Registration", zcncore.ConvertToValue(0)) - if err != nil { - return common.NewError("call_faucet_failed", "Failed to execute smart contract with err: "+err.Error()) - } - wg.Wait() - if !statusBar.success { - return common.NewError("call_faucet_failed", "Failed to execute smart contract with statusBar success failed") - } - statusBar.success = false - wg.Add(1) - err = txn.Verify() - if err != nil { - return common.NewError("call_faucet_failed", "Failed to verify smart contract with err: "+err.Error()) - } - wg.Wait() - if !statusBar.success { - return common.NewError("call_faucet_failed", "Failed to verify smart contract with statusBar success failed") - } - return nil -} - -func Transfer(token float64, clientID string) error { - wg := &sync.WaitGroup{} - statusBar := &ZCNStatus{wg: wg} - txn, err := zcncore.NewTransaction(statusBar, 0, 0) - if err != nil { - return common.NewError("call_transfer_failed", "Failed to create new transaction with err: "+err.Error()) - } - wg.Add(1) - err = txn.Send(clientID, zcncore.ConvertToValue(token), "Blobber delegate transfer") - if err != nil { - return common.NewError("call_transfer_failed", "Failed to send tokens with err: "+err.Error()) - } - wg.Wait() - if !statusBar.success { - return common.NewError("call_transfer_failed", "Failed to send tokens with statusBar success failed") - } - statusBar.success = false - wg.Add(1) - err = txn.Verify() - if err != nil { - return common.NewError("call_transfer_failed", "Failed to verify send transaction with err: "+err.Error()) - } - wg.Wait() - if !statusBar.success { - return common.NewError("call_transfer_failed", "Failed to verify send transaction with statusBar success failed") - } - return nil -} diff --git a/code/go/0chain.net/blobbercore/mock/ctx.go b/code/go/0chain.net/blobbercore/mock/ctx.go index b41344c2e..ac7195abf 100644 --- a/code/go/0chain.net/blobbercore/mock/ctx.go +++ b/code/go/0chain.net/blobbercore/mock/ctx.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" ) func SetupHandlerContext(ctx context.Context, r *http.Request, allocation string) context.Context { diff --git a/code/go/0chain.net/blobbercore/mock/init.go b/code/go/0chain.net/blobbercore/mock/init.go index e1c2fa978..624cb850c 100644 --- a/code/go/0chain.net/blobbercore/mock/init.go +++ b/code/go/0chain.net/blobbercore/mock/init.go @@ -6,9 +6,9 @@ import ( "net/http" "net/http/httptest" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/sdks" - "github.com/0chain/gosdk/sdks/blobber" + "github.com/0chain/gosdk_common/core/zcncrypto" + "github.com/0chain/gosdk_common/sdks" + "github.com/0chain/gosdk_common/sdks/blobber" ) const ( diff --git a/code/go/0chain.net/blobbercore/readmarker/protocol.go b/code/go/0chain.net/blobbercore/readmarker/protocol.go index e568f0070..51423fc89 100644 --- a/code/go/0chain.net/blobbercore/readmarker/protocol.go +++ b/code/go/0chain.net/blobbercore/readmarker/protocol.go @@ -6,12 +6,9 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/core/common" - zLogger "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "gorm.io/gorm" - - "go.uber.org/zap" ) type ReadRedeem struct { @@ -41,35 +38,7 @@ func (rme *ReadMarkerEntity) PendNumBlocks() (pendNumBlocks int64, err error) { // RedeemReadMarker redeems the read marker. func (rme *ReadMarkerEntity) RedeemReadMarker(ctx context.Context) (err error) { - tx, err := transaction.NewTransactionEntity() - if err != nil { - return common.NewErrorf("redeem_read_marker", "creating transaction: %v", err) - } - - sn := &ReadRedeem{ - ReadMarker: rme.LatestRM, - } - - err = tx.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.READ_REDEEM, sn, 0) - if err != nil { - zLogger.Logger.Info("Failed submitting read redeem", zap.Error(err)) - return common.NewErrorf("redeem_read_marker", "sending transaction: %v", err) - } - - time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second) - - var logHash = tx.Hash // keep transaction hash for error logs - tx, err = transaction.VerifyTransactionWithNonce(tx.Hash, tx.GetTransaction().GetTransactionNonce()) - if err != nil { - zLogger.Logger.Error("Error verifying the read redeem transaction", zap.Error(err), zap.String("txn", logHash)) - return common.NewErrorf("redeem_read_marker", "verifying transaction: %v", err) - } - - err = rme.UpdateStatus(ctx, tx.TransactionOutput, tx.Hash) - if err != nil { - return common.NewErrorf("redeem_read_marker", "updating read marker status: %v", err) - } - + // Depreciated return } diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go index b2df8007a..2fce04489 100644 --- a/code/go/0chain.net/blobbercore/readmarker/readmarker.go +++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go @@ -12,7 +12,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/encryption" zLogger "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "go.uber.org/zap" "gorm.io/gorm" ) diff --git a/code/go/0chain.net/blobbercore/util/json.go b/code/go/0chain.net/blobbercore/util/json.go index dbfce39ac..39afbdbc1 100644 --- a/code/go/0chain.net/blobbercore/util/json.go +++ b/code/go/0chain.net/blobbercore/util/json.go @@ -5,14 +5,15 @@ import ( "reflect" "strings" - "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk_common/zboxcore/fileref" ) // Validate unmarshalled data with tag-based rules // Example: -// struct { -// Name string `json:"name" validation:"required"` -// } +// +// struct { +// Name string `json:"name" validation:"required"` +// } func UnmarshalValidation(v interface{}) error { fields := reflect.ValueOf(v).Elem() diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 4bf3663dd..006dd33ff 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -9,7 +9,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/errors" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "go.uber.org/zap" ) diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index c33c30d07..79c16b5bf 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -3,17 +3,15 @@ package writemarker import ( "context" "fmt" - "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" + "github.com/0chain/gosdk_common/core/transaction" "go.uber.org/zap" ) @@ -25,7 +23,19 @@ type CommitConnection struct { ChainData []byte `json:"chain_data"` } -const timeGap = 180 +const ( + ADD_BLOBBER_SC_NAME = "add_blobber" + UPDATE_BLOBBER_SC_NAME = "update_blobber_settings" + ADD_VALIDATOR_SC_NAME = "add_validator" + CLOSE_CONNECTION_SC_NAME = "commit_connection" + READ_REDEEM = "read_redeem" + CHALLENGE_RESPONSE = "challenge_response" + BLOBBER_HEALTH_CHECK = "blobber_health_check" + FINALIZE_ALLOCATION = "finalize_allocation" + VALIDATOR_HEALTH_CHECK = "validator_health_check" + STORAGE_CONTRACT_ADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d7" + timeGap = 180 +) // VerifyMarker verify WriteMarker's hash and check allocation_root if it is unique func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *allocation.Allocation, co *allocation.AllocationChangeCollector, latestWM *WriteMarkerEntity) error { @@ -113,7 +123,7 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64) error { if len(wme.CloseTxnID) > 0 { - t, err := transaction.VerifyTransaction(wme.CloseTxnID, chain.GetServerChain()) + t, err := transaction.VerifyTransaction(wme.CloseTxnID) if err == nil { wme.Status = Committed wme.StatusMessage = t.TransactionOutput @@ -123,19 +133,14 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64) } } - txn, err := transaction.NewTransactionEntity() - if err != nil { - wme.StatusMessage = "Error creating transaction entity. " + err.Error() - if err := wme.UpdateStatus(ctx, Failed, "Error creating transaction entity. "+err.Error(), "", startSeq, wme.Sequence); err != nil { - Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) - } - return err - } - + var out, hash string + var nonce int64 + txn := &transaction.Transaction{} sn := &CommitConnection{} sn.AllocationRoot = wme.WM.AllocationRoot sn.PrevAllocationRoot = wme.WM.PreviousAllocationRoot sn.WriteMarker = &wme.WM + var err error err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { sn.ChainData, err = GetMarkersForChain(ctx, wme.WM.AllocationID, startSeq, wme.Sequence-1) return err @@ -148,10 +153,9 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64) return err } - if sn.AllocationRoot == sn.PrevAllocationRoot { + if sn.AllocationRoot == sn.PrevAllocationRoot && wme.WM.Version != MARKER_VERSION { // get nonce of prev WM - var prevWM *WriteMarkerEntity - prevWM, err = GetPreviousWM(ctx, sn.AllocationRoot, wme.WM.Timestamp) + _, err = GetPreviousWM(ctx, sn.AllocationRoot, wme.WM.Timestamp) if err != nil { wme.StatusMessage = "Error getting previous write marker. " + err.Error() if err := wme.UpdateStatus(ctx, Failed, "Error getting previous write marker. "+err.Error(), "", startSeq, wme.Sequence); err != nil { @@ -159,10 +163,13 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64) } return err } - err = txn.ExecuteRollbackWM(transaction.STORAGE_CONTRACT_ADDRESS, transaction.CLOSE_CONNECTION_SC_NAME, sn, 0, prevWM.CloseTxnNonce) - } else { - err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.CLOSE_CONNECTION_SC_NAME, sn, 0) + } + + hash, out, nonce, txn, err = transaction.SmartContractTxn(STORAGE_CONTRACT_ADDRESS, transaction.SmartContractTxnData{ + Name: CLOSE_CONNECTION_SC_NAME, + InputArgs: sn, + }, true) if err != nil { Logger.Error("Failed during sending close connection to the miner. ", zap.String("err:", err.Error())) wme.Status = Failed @@ -173,23 +180,11 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64) return err } - time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second) - t, err := transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce()) wme.CloseTxnID = txn.Hash - wme.CloseTxnNonce = txn.GetTransaction().GetTransactionNonce() - if err != nil { - Logger.Error("Error verifying the close connection transaction", zap.String("err:", err.Error()), zap.String("txn", txn.Hash)) - wme.Status = Failed - wme.StatusMessage = "Error verifying the close connection transaction." + err.Error() - // TODO Is this single try? - if err := wme.UpdateStatus(ctx, Failed, "Error verifying the close connection transaction."+err.Error(), txn.Hash, startSeq, wme.Sequence); err != nil { - Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err)) - } - return err - } + wme.CloseTxnNonce = nonce wme.Status = Committed - wme.StatusMessage = t.TransactionOutput - err = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash, startSeq, wme.Sequence) + wme.StatusMessage = out + err = wme.UpdateStatus(ctx, Committed, out, hash, startSeq, wme.Sequence) return err } diff --git a/code/go/0chain.net/blobbercore/zcn/query.go b/code/go/0chain.net/blobbercore/zcn/query.go index 7b122ca77..1f00f1eb6 100644 --- a/code/go/0chain.net/blobbercore/zcn/query.go +++ b/code/go/0chain.net/blobbercore/zcn/query.go @@ -6,30 +6,23 @@ import ( "fmt" "sync" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk/zboxcore/sdk" + "github.com/0chain/gosdk_common/zcncore" ) var ErrBlobberNotFound = errors.New("blobber is not found on chain") // GetBlobber try to get blobber info from chain. -func GetBlobber(blobberID string) (*zcncore.Blobber, error) { - cb := &getBlobberCallback{} - cb.wg.Add(1) - if err := zcncore.GetBlobber(blobberID, cb); err != nil { - cb.wg.Done() +func GetBlobber(blobberID string) (*sdk.Blobber, error) { + var ( + blobber *sdk.Blobber + err error + ) + if blobber, err = sdk.GetBlobber(blobberID); err != nil { return nil, err } - cb.wg.Wait() - if cb.Error != nil { - return nil, cb.Error - } - - if cb.Blobber == nil { - return nil, ErrBlobberNotFound - } - - return cb.Blobber, nil + return blobber, nil } diff --git a/code/go/0chain.net/core/chain/entity.go b/code/go/0chain.net/core/chain/entity.go index 7360a4f93..4c82c3d13 100644 --- a/code/go/0chain.net/core/chain/entity.go +++ b/code/go/0chain.net/core/chain/entity.go @@ -23,13 +23,13 @@ func GetServerChain() *Chain { /*Chain - data structure that holds the chain data*/ type Chain struct { - ID string - Version string - CreationDate common.Timestamp - OwnerID string - ParentChainID string - BlockWorker string - + ID string + Version string + CreationDate common.Timestamp + OwnerID string + ParentChainID string + BlockWorker string + ZauthServer string GenesisBlockHash string } @@ -44,12 +44,13 @@ func (c *Chain) Validate(ctx context.Context) error { return nil } -//NewChainFromConfig - create a new chain from config +// NewChainFromConfig - create a new chain from config func NewChainFromConfig() *Chain { chain := Provider() chain.ID = common.ToKey(config.Configuration.ChainID) chain.OwnerID = viper.GetString("server_chain.owner") chain.BlockWorker = viper.GetString("block_worker") + chain.ZauthServer = viper.GetString("zauth_server") return chain } diff --git a/code/go/0chain.net/core/encryption/keys.go b/code/go/0chain.net/core/encryption/keys.go index bb652f1bf..58327656c 100644 --- a/code/go/0chain.net/core/encryption/keys.go +++ b/code/go/0chain.net/core/encryption/keys.go @@ -9,12 +9,14 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/config" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk_common/core/zcncrypto" "github.com/herumi/bls-go-binary/bls" ) -/*ReadKeys - reads a publicKey and a privateKey from a Reader. -They are assumed to be in two separate lines one followed by the other*/ +/* +ReadKeys - reads a publicKey and a privateKey from a Reader. +They are assumed to be in two separate lines one followed by the other +*/ func ReadKeys(reader io.Reader) (publicKey, privateKey, publicIp, port string) { scanner := bufio.NewScanner(reader) scanner.Scan() @@ -45,13 +47,14 @@ func Verify(publicKey, signature, hash string) (bool, error) { } // If input is normal herumi/bls public key, it returns it immmediately. -// So this is completely backward compatible with herumi/bls. +// +// So this is completely backward compatible with herumi/bls. +// // If input is MIRACL public key, convert it to herumi/bls public key. // // This is an example of the raw public key we expect from MIRACL var miraclExamplePK = `0418a02c6bd223ae0dfda1d2f9a3c81726ab436ce5e9d17c531ff0a385a13a0b491bdfed3a85690775ee35c61678957aaba7b1a1899438829f1dc94248d87ed36817f6dfafec19bfa87bf791a4d694f43fec227ae6f5a867490e30328cac05eaff039ac7dfc3364e851ebd2631ea6f1685609fc66d50223cc696cb59ff2fee47ac` -// // This is an example of the same MIRACL public key serialized with ToString(). // pk ([1bdfed3a85690775ee35c61678957aaba7b1a1899438829f1dc94248d87ed368,18a02c6bd223ae0dfda1d2f9a3c81726ab436ce5e9d17c531ff0a385a13a0b49],[039ac7dfc3364e851ebd2631ea6f1685609fc66d50223cc696cb59ff2fee47ac,17f6dfafec19bfa87bf791a4d694f43fec227ae6f5a867490e30328cac05eaff]) func MiraclToHerumiPK(pk string) string { diff --git a/code/go/0chain.net/core/encryption/keys_test.go b/code/go/0chain.net/core/encryption/keys_test.go index b0a71afc3..1b411a0e3 100644 --- a/code/go/0chain.net/core/encryption/keys_test.go +++ b/code/go/0chain.net/core/encryption/keys_test.go @@ -3,8 +3,8 @@ package encryption import ( "testing" - "github.com/0chain/gosdk/core/sys" - "github.com/0chain/gosdk/zboxcore/client" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/core/sys" "github.com/herumi/bls-go-binary/bls" "github.com/stretchr/testify/require" ) @@ -12,7 +12,8 @@ import ( func TestSignatureVerify(t *testing.T) { allocationId := "4f928c7857fabb5737347c42204eea919a4777f893f35724f563b932f64e2367" walletConfig := "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}" - require.NoError(t, client.PopulateClient(walletConfig, "bls0chain")) + _, err := client.PopulateClient(walletConfig, "bls0chain") + require.NoError(t, err) sig, serr := client.Sign(allocationId) require.Nil(t, serr) require.NotNil(t, sig) diff --git a/code/go/0chain.net/core/node/context.go b/code/go/0chain.net/core/node/context.go index 083851263..d5d0bb97c 100644 --- a/code/go/0chain.net/core/node/context.go +++ b/code/go/0chain.net/core/node/context.go @@ -3,7 +3,7 @@ package node import ( "context" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" ) const SELF_NODE constants.ContextKey = "SELF_NODE" diff --git a/code/go/0chain.net/core/node/self_node.go b/code/go/0chain.net/core/node/self_node.go index c58692f59..047980d91 100644 --- a/code/go/0chain.net/core/node/self_node.go +++ b/code/go/0chain.net/core/node/self_node.go @@ -4,9 +4,10 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/config" - "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk_common/core/zcncrypto" "golang.org/x/crypto/sha3" ) diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go index 00908352c..c0c80e7d6 100644 --- a/code/go/0chain.net/core/transaction/entity.go +++ b/code/go/0chain.net/core/transaction/entity.go @@ -1,20 +1,10 @@ package transaction import ( - "encoding/json" "sync" "time" - "github.com/0chain/gosdk/core/transaction" - - "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "go.uber.org/zap" - - "github.com/0chain/gosdk/zcncore" - - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/node" ) var ( @@ -22,25 +12,6 @@ var ( last50TransactionsMutex sync.Mutex ) -// Transaction entity that encapsulates the transaction related data and meta data -type Transaction struct { - Hash string `json:"hash,omitempty"` - Version string `json:"version,omitempty"` - ClientID string `json:"client_id,omitempty"` - PublicKey string `json:"public_key,omitempty"` - ToClientID string `json:"to_client_id,omitempty"` - ChainID string `json:"chain_id,omitempty"` - TransactionData string `json:"transaction_data,omitempty"` - Value int64 `json:"transaction_value,omitempty"` - Signature string `json:"signature,omitempty"` - CreationDate common.Timestamp `json:"creation_date,omitempty"` - TransactionType int `json:"transaction_type,omitempty"` - TransactionOutput string `json:"transaction_output,omitempty"` - OutputHash string `json:"txn_output_hash"` - zcntxn zcncore.TransactionScheme - wg *sync.WaitGroup -} - type SmartContractTxnData struct { Name string `json:"name"` InputArgs interface{} `json:"input"` @@ -82,19 +53,20 @@ type BlobberAllocation struct { } type StorageAllocation struct { - ID string `json:"id"` - Tx string `json:"tx"` - OwnerPublicKey string `json:"owner_public_key"` - OwnerID string `json:"owner_id"` - Size int64 `json:"size"` - UsedSize int64 `json:"used_size"` - Expiration common.Timestamp `json:"expiration_date"` - BlobberDetails []*BlobberAllocation `json:"blobber_details"` - Finalized bool `json:"finalized"` - TimeUnit time.Duration `json:"time_unit"` - WritePool uint64 `json:"write_pool"` - FileOptions uint16 `json:"file_options"` - StartTime common.Timestamp `json:"start_time"` + ID string `json:"id"` + Tx string `json:"tx"` + OwnerPublicKey string `json:"owner_public_key"` + OwnerID string `json:"owner_id"` + Size int64 `json:"size"` + UsedSize int64 `json:"used_size"` + Expiration common.Timestamp `json:"expiration_date"` + BlobberDetails []*BlobberAllocation `json:"blobber_details"` + Finalized bool `json:"finalized"` + TimeUnit time.Duration `json:"time_unit"` + WritePool uint64 `json:"write_pool"` + FileOptions uint16 `json:"file_options"` + StartTime common.Timestamp `json:"start_time"` + OwnerSigningPublicKey string `json:"owner_signing_public_key"` DataShards int64 `json:"data_shards"` ParityShards int64 `json:"parity_shards"` @@ -121,216 +93,6 @@ const ( const STORAGE_CONTRACT_ADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d7" -func NewTransactionEntity() (*Transaction, error) { - txn := &Transaction{} - txn.Version = "1.0" - txn.ClientID = node.Self.ID - txn.CreationDate = common.Now() - txn.ChainID = chain.GetServerChain().ID - txn.PublicKey = node.Self.PublicKey - txn.wg = &sync.WaitGroup{} - zcntxn, err := zcncore.NewTransaction(txn, 0, 0) - if err != nil { - return nil, err - } - txn.zcntxn = zcntxn - return txn, nil -} - -func (t *Transaction) GetTransaction() zcncore.TransactionScheme { - return t.zcntxn -} - -func (t *Transaction) ExecuteSmartContract(address, methodName string, input interface{}, val uint64) error { - t.wg.Add(1) - - sn := transaction.SmartContractTxnData{Name: methodName, InputArgs: input} - snBytes, err := json.Marshal(sn) - if err != nil { - return err - } - - updateLast50Transactions(string(snBytes)) - - nonce := monitor.getNextUnusedNonce() - if err := t.zcntxn.SetTransactionNonce(nonce); err != nil { - logging.Logger.Error("Failed to set nonce.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", nonce), - zap.Any("error", err)) - } - - logging.Logger.Info("Transaction nonce set.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", nonce)) - - _, err = t.zcntxn.ExecuteSmartContract(address, methodName, input, uint64(val)) - if err != nil { - t.wg.Done() - logging.Logger.Error("Failed to execute SC.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce()), - zap.Any("error", err)) - monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce()) - return err - } - - t.wg.Wait() - - t.Hash = t.zcntxn.GetTransactionHash() - if len(t.zcntxn.GetTransactionError()) > 0 { - logging.Logger.Error("Failed to submit SC.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce()), - zap.Any("error", t.zcntxn.GetTransactionError())) - monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce()) - return common.NewError("transaction_send_error", t.zcntxn.GetTransactionError()) - } - return nil -} - -func (t *Transaction) ExecuteRollbackWM(address, methodName string, input interface{}, val uint64, prevNonce int64) error { - t.wg.Add(1) - - sn := transaction.SmartContractTxnData{Name: methodName, InputArgs: input} - snBytes, err := json.Marshal(sn) - if err != nil { - return err - } - - updateLast50Transactions(string(snBytes)) - - nonce := monitor.getNextUnusedNonce() - if nonce < prevNonce { - t.wg.Done() - logging.Logger.Error("Failed to set nonce as prevNonce is greater", - zap.Any("nonce", nonce), - zap.Any("prevNonce", prevNonce), - ) - monitor.recordFailedNonce(nonce) - return common.NewError("transaction_send_error", "Failed to set nonce as prevNonce is greater") - } - if err := t.zcntxn.SetTransactionNonce(nonce); err != nil { - logging.Logger.Error("Failed to set nonce.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", nonce), - zap.Any("error", err)) - } - - logging.Logger.Info("Transaction nonce set.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", nonce)) - - _, err = t.zcntxn.ExecuteSmartContract(address, methodName, input, uint64(val)) - if err != nil { - t.wg.Done() - logging.Logger.Error("Failed to execute SC.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce()), - zap.Any("error", err)) - monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce()) - return err - } - - t.wg.Wait() - - t.Hash = t.zcntxn.GetTransactionHash() - if len(t.zcntxn.GetTransactionError()) > 0 { - logging.Logger.Error("Failed to submit SC.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce()), - zap.Any("error", t.zcntxn.GetTransactionError())) - monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce()) - return common.NewError("transaction_send_error", t.zcntxn.GetTransactionError()) - } - return nil -} - -func (t *Transaction) Verify() error { - if err := t.zcntxn.SetTransactionHash(t.Hash); err != nil { - monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce()) - logging.Logger.Error("Failed to set txn hash.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce()), - zap.Any("error", err)) - return err - } - t.wg.Add(1) - err := t.zcntxn.Verify() - if err != nil { - t.wg.Done() - logging.Logger.Error("Failed to start txn verification.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce()), - zap.Any("error", err)) - monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce()) - return err - } - t.wg.Wait() - if len(t.zcntxn.GetVerifyError()) > 0 { - logging.Logger.Error("Failed to verify txn.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce()), - zap.Any("error", t.zcntxn.GetVerifyError()), - zap.Any("verify_output", t.zcntxn.GetVerifyOutput())) - monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce()) - return common.NewError("transaction_verify_error", t.zcntxn.GetVerifyError()) - } else { - logging.Logger.Info("Successful txn verification.", - zap.Any("hash", t.zcntxn.GetTransactionHash()), - zap.Any("nonce", t.zcntxn.GetTransactionNonce())) - monitor.recordSuccess(t.zcntxn.GetTransactionNonce()) - } - - output := t.zcntxn.GetVerifyOutput() - - var objmap map[string]json.RawMessage - err = json.Unmarshal([]byte(output), &objmap) - if err != nil { - // it is a plain error message from blockchain. The format is `error_code: error message`. eg verify_challenge: could not find challenge, value not present - // so it is impossible to decode as map[string]json.RawMessage. - return common.NewError("transaction_verify_error", string(output)) - } - - err = json.Unmarshal(objmap["txn"], t) - if err != nil { - var confirmation map[string]json.RawMessage - err = json.Unmarshal(objmap["confirmation"], &confirmation) - if err != nil { - return common.NewError("transaction_verify_error", "Error unmarshaling verify output->confirmation: "+string(output)+" "+err.Error()) - } - err = json.Unmarshal(confirmation["txn"], t) - if err != nil { - return common.NewError("transaction_verify_error", "Error unmarshaling verify output->confirmation->txn: "+string(output)+" "+err.Error()) - } - } - return nil -} - -// func (t *Transaction) ComputeHashAndSign() error { -// hashdata := fmt.Sprintf("%v:%v:%v:%v:%v", t.CreationDate, t.ClientID, -// t.ToClientID, t.Value, encryption.Hash(t.TransactionData)) -// t.Hash = encryption.Hash(hashdata) -// var err error -// t.Signature, err = node.Self.Sign(t.Hash) -// if err != nil { -// return err -// } -// return nil -// } - -func (t *Transaction) OnTransactionComplete(zcntxn *zcncore.Transaction, status int) { - t.wg.Done() -} - -func (t *Transaction) OnVerifyComplete(zcntxn *zcncore.Transaction, status int) { - t.wg.Done() -} - -func (t *Transaction) OnAuthComplete(zcntxn *zcncore.Transaction, status int) { - -} - func updateLast50Transactions(data string) { last50TransactionsMutex.Lock() defer last50TransactionsMutex.Unlock() diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index 52feee60c..f38f0bd06 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -1,9 +1,8 @@ package transaction import ( - "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/zboxcore/zboxutil" + "github.com/0chain/gosdk_common/core/screstapi" ) const TXN_SUBMIT_URL = "v1/transaction/put" @@ -16,39 +15,8 @@ const ( ) var ErrNoTxnDetail = common.NewError("missing_transaction_detail", "No transaction detail was found on any of the sharders") -var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string) ([]byte, error) = MakeSCRestAPICallNoHandler +var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string, options ...string) ([]byte, error) = MakeSCRestAPICallNoHandler -func MakeSCRestAPICallNoHandler(address string, path string, params map[string]string) ([]byte, error) { - return zboxutil.MakeSCRestAPICall(address, path, params, nil) -} - -func VerifyTransaction(txnHash string, chain *chain.Chain) (*Transaction, error) { - txn, err := NewTransactionEntity() - if err != nil { - return nil, err - } - - txn.Hash = txnHash - err = txn.Verify() - if err != nil { - return nil, err - } - return txn, nil -} - -// VerifyTransactionWithNonce verifies a transaction with known nonce. -func VerifyTransactionWithNonce(txnHash string, nonce int64) (*Transaction, error) { - txn, err := NewTransactionEntity() - if err != nil { - return nil, err - } - - txn.Hash = txnHash - _ = txn.zcntxn.SetTransactionNonce(nonce) - - err = txn.Verify() - if err != nil { - return nil, err - } - return txn, nil +func MakeSCRestAPICallNoHandler(address string, path string, params map[string]string, options ...string) ([]byte, error) { + return screstapi.MakeSCRestAPICall(address, path, params, options...) } diff --git a/code/go/0chain.net/core/transaction/nonce.go b/code/go/0chain.net/core/transaction/nonce.go index 44013c7a6..403031e27 100644 --- a/code/go/0chain.net/core/transaction/nonce.go +++ b/code/go/0chain.net/core/transaction/nonce.go @@ -5,7 +5,8 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/zcncore" "go.uber.org/zap" ) @@ -88,23 +89,7 @@ func (m *nonceMonitor) refreshFromBalance() { // sync lock not necessary, this is expected to be called within a synchronized function. m.shouldRefreshFromBalance = false - cb := &getNonceCallBack{waitCh: make(chan struct{})} - if err := zcncore.GetNonce(cb); err != nil { - return - } - - <-cb.waitCh - - newNonce := int64(0) - if cb.hasError { - logging.Logger.Info("Couldn't get nonce from remote, use 0") - newNonce = int64(0) - } else { - logging.Logger.Info("Got nonce from balance.", zap.Any("nonce", cb.nonce), zap.Any("highestSuccess", m.highestSuccess)) - newNonce = cb.nonce - } - - m.highestSuccess = newNonce + m.highestSuccess = client.Nonce() m.failed = make(map[int64]int64) m.used = make(map[int64]time.Time) diff --git a/code/go/0chain.net/validator/main.go b/code/go/0chain.net/validator/main.go index bf5ca7f12..330387e93 100644 --- a/code/go/0chain.net/validator/main.go +++ b/code/go/0chain.net/validator/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "flag" "fmt" "log" @@ -21,12 +22,13 @@ import ( . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - "github.com/0chain/blobber/code/go/0chain.net/core/util" "github.com/0chain/blobber/code/go/0chain.net/validatorcore/config" "github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage" + coreTxn "github.com/0chain/gosdk_common/core/transaction" "github.com/0chain/gosdk/zboxcore/sdk" - "github.com/0chain/gosdk/zcncore" + "github.com/0chain/gosdk_common/core/client" + "github.com/0chain/gosdk_common/zcncore" "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/spf13/viper" @@ -202,28 +204,26 @@ func RegisterValidator() { } for { - txn, err := storage.GetProtocolImpl().RegisterValidator(common.GetRootContext()) + sn := &transaction.StorageNode{} + sn.ID = node.Self.ID + sn.BaseURL = node.Self.GetURLBase() + sn.StakePoolSettings.DelegateWallet = config.Configuration.DelegateWallet + sn.StakePoolSettings.NumDelegates = config.Configuration.NumDelegates + sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge + + hash, out, _, _, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{ + Name: transaction.ADD_VALIDATOR_SC_NAME, + InputArgs: sn, + }, true) if err != nil { - Logger.Error("Error registering validator", zap.Any("err", err)) + Logger.Error("Add validator transaction could not be verified", zap.Any("err", err), zap.String("txn.Hash", hash)) continue } - time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second) - txnVerified := false - verifyRetries := 0 - for verifyRetries < util.MAX_RETRIES { - time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second) - t, err := transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce()) - if err == nil { - Logger.Info("Transaction for adding validator accepted and verified", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput)) - go handler.StartHealthCheck(common.GetRootContext(), common.ProviderTypeValidator) - return - } - verifyRetries++ - } - if !txnVerified { - Logger.Error("Add validator transaction could not be verified", zap.Any("err", err), zap.String("txn.Hash", txn.Hash)) - } + Logger.Info("Transaction for adding validator accepted and verified", zap.String("txn_hash", hash), zap.Any("txn_output", out)) + + go handler.StartHealthCheck(common.GetRootContext(), common.ProviderTypeValidator) + break } } @@ -232,17 +232,16 @@ func SetupValidatorOnBC(logDir string) error { var logName = logDir + "/validator.log" zcncore.SetLogFile(logName, false) zcncore.SetLogLevel(3) - if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil { - return err - } - if err := zcncore.SetWalletInfo(node.Self.GetWalletString(), false); err != nil { + err := client.InitSDK("{}", serverChain.BlockWorker, config.Configuration.ChainID, config.Configuration.SignatureScheme, int64(0), false) + if err != nil { return err } - var blob []string - if err := sdk.InitStorageSDK(node.Self.GetWalletString(), serverChain.BlockWorker, - config.Configuration.ChainID, config.Configuration.SignatureScheme, blob, int64(0)); err != nil { + + err = zcncore.SetGeneralWalletInfo(node.Self.GetWalletString(), config.Configuration.SignatureScheme) + if err != nil { return err } + go RegisterValidator() return nil } @@ -254,7 +253,7 @@ func HomePageHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "
Working on the chain: %v
\n", mc.ID) fmt.Fprintf(w, "
I am a validator with
  • id:%v
  • public_key:%v
  • build_tag:%v
\n", node.Self.ID, node.Self.PublicKey, build.BuildTag) fmt.Fprintf(w, "
Miners ...\n") - network := zcncore.GetNetwork() + network, _ := client.GetNetwork(context.Background()) for _, miner := range network.Miners { fmt.Fprintf(w, "%v\n", miner) } diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler.go b/code/go/0chain.net/validatorcore/storage/challenge_handler.go index e88aaba8f..53999cf32 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler.go @@ -13,6 +13,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "go.uber.org/zap" "golang.org/x/crypto/sha3" @@ -41,7 +42,7 @@ func challengeHandler(ctx context.Context, r *http.Request) (interface{}, error) time.Sleep(1 * time.Second) - allocationObj, err := GetProtocolImpl().VerifyAllocationTransaction(ctx, challengeObj.AllocationID) + allocationObj, err := requestAllocation(challengeObj.AllocationID) if err != nil { logging.Logger.Error("Error verifying the allocation from BC", zap.String("allocation_id", challengeObj.AllocationID), zap.Error(err)) return nil, common.NewError("invalid_parameters", "Allocation could not be verified. "+err.Error()) @@ -58,6 +59,33 @@ func challengeHandler(ctx context.Context, r *http.Request) (interface{}, error) return ValidValidationTicket(challengeObj, challengeRequest.ChallengeID, challengeHash) } +func requestAllocation(allocID string) (allocation *Allocation, err error) { + var b []byte + b, err = transaction.MakeSCRestAPICall( + transaction.STORAGE_CONTRACT_ADDRESS, + "/allocation", + map[string]string{"allocation": allocID}) + if err != nil { + return + } + sa := new(transaction.StorageAllocation) + err = json.Unmarshal(b, sa) + if err != nil { + return + } + allocation = &Allocation{ + ID: sa.ID, + DataShards: sa.DataShards, + ParityShards: sa.ParityShards, + Size: sa.Size, + Expiration: sa.Expiration, + Owner: sa.OwnerID, + OwnerPublicKey: sa.OwnerPublicKey, + OwnerSigningPublicKey: sa.OwnerSigningPublicKey, + } + return +} + func NewChallengeRequest(r *http.Request) (*ChallengeRequest, string, error) { if r.Method == "GET" { return nil, "", common.NewError("invalid_method", "Invalid method used for the upload URL. Use multi-part form POST instead") diff --git a/code/go/0chain.net/validatorcore/storage/context.go b/code/go/0chain.net/validatorcore/storage/context.go index 86a2acb5d..7b8449ea0 100644 --- a/code/go/0chain.net/validatorcore/storage/context.go +++ b/code/go/0chain.net/validatorcore/storage/context.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" ) func SetupContext(handler common.JSONResponderF) common.JSONResponderF { diff --git a/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go b/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go index 6fb11006b..a4fa31ad7 100644 --- a/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go +++ b/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go @@ -8,7 +8,7 @@ import ( "net/http" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk_common/constants" "github.com/gorilla/mux" ) diff --git a/code/go/0chain.net/validatorcore/storage/models.go b/code/go/0chain.net/validatorcore/storage/models.go index ba47ef2f8..00746d2c9 100644 --- a/code/go/0chain.net/validatorcore/storage/models.go +++ b/code/go/0chain.net/validatorcore/storage/models.go @@ -11,7 +11,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage/writemarker" - "github.com/0chain/gosdk/core/util" + "github.com/0chain/gosdk_common/core/util" "github.com/mitchellh/mapstructure" @@ -263,14 +263,15 @@ func (op *ObjectPath) Verify(allocationID string, challengeRand int64) error { } type Allocation struct { - ID string `json:"id"` - DataShards int `json:"data_shards"` - ParityShards int `json:"parity_shards"` - Size int64 `json:"size"` - UsedSize int64 `json:"used_size"` - Expiration common.Timestamp `json:"expiration_date"` - Owner string `json:"owner_id"` - OwnerPublicKey string `json:"owner_public_key"` + ID string `json:"id"` + DataShards int64 `json:"data_shards"` + ParityShards int64 `json:"parity_shards"` + Size int64 `json:"size"` + UsedSize int64 `json:"used_size"` + Expiration common.Timestamp `json:"expiration_date"` + Owner string `json:"owner_id"` + OwnerPublicKey string `json:"owner_public_key"` + OwnerSigningPublicKey string `json:"owner_signing_public_key"` } type ChallengeProof struct { diff --git a/code/go/0chain.net/validatorcore/storage/models_test.go b/code/go/0chain.net/validatorcore/storage/models_test.go index 221053cdb..eef445acb 100644 --- a/code/go/0chain.net/validatorcore/storage/models_test.go +++ b/code/go/0chain.net/validatorcore/storage/models_test.go @@ -10,8 +10,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage" - "github.com/0chain/gosdk/core/util" - "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk_common/core/util" + "github.com/0chain/gosdk_common/core/zcncrypto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" diff --git a/code/go/0chain.net/validatorcore/storage/protocol.go b/code/go/0chain.net/validatorcore/storage/protocol.go index 521d3ee01..1871a008a 100644 --- a/code/go/0chain.net/validatorcore/storage/protocol.go +++ b/code/go/0chain.net/validatorcore/storage/protocol.go @@ -4,17 +4,13 @@ import ( "context" "encoding/json" "sync" - "time" "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - "github.com/0chain/blobber/code/go/0chain.net/validatorcore/config" - "github.com/0chain/gosdk/constants" - "go.uber.org/zap" + "github.com/0chain/gosdk_common/constants" ) const CHUNK_SIZE = 64 * 1024 @@ -67,19 +63,6 @@ func GetProtocolImpl() *ValidatorProtocolImpl { // return txn.Hash, nil // } -func (sp *ValidatorProtocolImpl) VerifyAllocationTransaction(ctx context.Context, allocationID string) (*Allocation, error) { - t, err := transaction.VerifyTransaction(allocationID, sp.ServerChain) - if err != nil { - return nil, common.NewError("invalid_allocation", "Invalid Allocation id. Allocation not found in blockchain. "+err.Error()) - } - var allocationObj Allocation - err = json.Unmarshal([]byte(t.TransactionOutput), &allocationObj) - if err != nil { - return nil, common.NewError("transaction_output_decode_error", "Error decoding the allocation transaction output."+err.Error()) - } - return &allocationObj, nil -} - func (sp *ValidatorProtocolImpl) VerifyChallengeTransaction(ctx context.Context, challengeRequest *ChallengeRequest) (*Challenge, error) { blobberID := ctx.Value(constants.ContextKeyClient).(string) if blobberID == "" { @@ -125,28 +108,3 @@ func (wb *WalletCallback) OnWalletCreateComplete(status int, wallet, err string) wb.err = err wb.wg.Done() } - -func (sp *ValidatorProtocolImpl) RegisterValidator(ctx context.Context) (*transaction.Transaction, error) { - time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second) - - txn, err := transaction.NewTransactionEntity() - if err != nil { - return nil, err - } - - sn := &transaction.StorageNode{} - sn.ID = node.Self.ID - sn.BaseURL = node.Self.GetURLBase() - sn.StakePoolSettings.DelegateWallet = config.Configuration.DelegateWallet - sn.StakePoolSettings.NumDelegates = config.Configuration.NumDelegates - sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge - - logging.Logger.Info("Adding validator to the blockchain.") - err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.ADD_VALIDATOR_SC_NAME, sn, 0) - if err != nil { - logging.Logger.Info("Failed during registering validator to the mining network", zap.String("err:", err.Error())) - return nil, err - } - - return txn, nil -} diff --git a/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go b/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go index 7f02f46a1..86a4ef3bf 100644 --- a/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go +++ b/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go @@ -9,7 +9,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage/writemarker" - "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk_common/core/zcncrypto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go.mod b/go.mod index 310b7f48c..d60d44d09 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/0chain/blobber -go 1.21 +go 1.22.0 + +toolchain go1.23.1 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.17.0-RC1 + github.com/0chain/gosdk v1.19.0-RC0 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 @@ -19,16 +21,16 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/selvatico/go-mocket v1.0.7 github.com/spf13/viper v1.16.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.17.0 - golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 + golang.org/x/crypto v0.21.0 + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect google.golang.org/grpc v1.56.2 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.33.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gorm.io/datatypes v1.2.0 gorm.io/driver/postgres v1.5.2 @@ -37,6 +39,7 @@ require ( ) require ( + github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f github.com/lithammer/shortuuid/v3 v3.0.7 golang.org/x/sync v0.7.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc @@ -69,15 +72,17 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect github.com/aws/smithy-go v1.19.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/hack-pad/go-webworkers v0.1.0 // indirect github.com/hack-pad/safejs v0.1.1 // indirect - github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 // indirect + github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.51.0 // indirect + github.com/x448/float16 v0.8.4 // indirect ) require ( - github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 + github.com/0chain/common v1.18.3 github.com/Luzifer/go-openssl/v3 v3.1.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/btcsuite/btcd v0.23.4 // indirect diff --git a/go.sum b/go.sum index 7f02eaf24..0864bb37b 100644 --- a/go.sum +++ b/go.sum @@ -36,12 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEsT2XtRu4X7GfBiMnz9dYvWYs9V0B4= -github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= +github.com/0chain/common v1.18.3 h1:42dYOv2KyMTSanuS67iDtfv+ErbSRqR8NJ3MG72MwaI= +github.com/0chain/common v1.18.3/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.17.0-RC1 h1:D3OwgmfkqW/2FR+JtjmDx+KhlCINXL5/ZWNMbbPh4lc= -github.com/0chain/gosdk v1.17.0-RC1/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg= +github.com/0chain/gosdk v1.19.0-RC0 h1:PhHsvfEBJw9ofEFGWKqJ7UID7qMfl1LrWl2GyhIxjqE= +github.com/0chain/gosdk v1.19.0-RC0/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw= +github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f h1:H3Edk0y4Q8GEsPAbIsmn+WKiTC/uA4L/YuX0x/3Adz0= +github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f/go.mod h1:aGBabvKYk/B6G6bEBI8j1cBaWY2is9sCii88Ly6VeCU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -147,7 +149,6 @@ github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -162,14 +163,18 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= -github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= -github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -229,12 +234,14 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= -github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -484,8 +491,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/herumi/bls-go-binary v1.31.0 h1:L1goQ2tMtGgpXCg5AwHAdJQpLs/pfnWWEc3Wog6OhmI= github.com/herumi/bls-go-binary v1.31.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= -github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 h1:Z6Mu2JCsW2hbqx91L0HNPRPQ10RyAFvPocQHlrRo1Jk= -github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk= +github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c h1:lDSIbcLu5TdT+uwb4wPzZgo1pQvKjP/tArL5QKjDJdI= +github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= @@ -782,8 +789,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -793,8 +801,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -826,6 +834,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -905,8 +915,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -918,8 +928,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -993,8 +1003,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1087,8 +1097,8 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1287,8 +1297,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 99a44a4c1dcb1cdd55f532b38a03cfba064c86d0 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Mon, 24 Feb 2025 18:17:00 +0530 Subject: [PATCH 92/98] conflict fix --- code/go/0chain.net/blobbercore/handler/handler_common.go | 2 +- go.mod | 2 +- go.sum | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler_common.go b/code/go/0chain.net/blobbercore/handler/handler_common.go index f9e2c41a2..340a82378 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_common.go +++ b/code/go/0chain.net/blobbercore/handler/handler_common.go @@ -13,7 +13,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/gosdk/core/client" + "github.com/0chain/gosdk_common/core/client" "go.uber.org/zap" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" diff --git a/go.mod b/go.mod index d60d44d09..814010481 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.1 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.19.0-RC0 + github.com/0chain/gosdk v1.17.0-RC6 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 diff --git a/go.sum b/go.sum index 0864bb37b..bd8c07625 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v1.18.3 h1:42dYOv2KyMTSanuS67iDtfv+ErbSRqR8NJ3MG72MwaI= github.com/0chain/common v1.18.3/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.19.0-RC0 h1:PhHsvfEBJw9ofEFGWKqJ7UID7qMfl1LrWl2GyhIxjqE= -github.com/0chain/gosdk v1.19.0-RC0/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw= +github.com/0chain/gosdk v1.17.0-RC6 h1:NB4pfu0VbVWfQQEfnFqn5inaiQuyhSaUW7VwgH3LW8U= +github.com/0chain/gosdk v1.17.0-RC6/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg= github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f h1:H3Edk0y4Q8GEsPAbIsmn+WKiTC/uA4L/YuX0x/3Adz0= github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f/go.mod h1:aGBabvKYk/B6G6bEBI8j1cBaWY2is9sCii88Ly6VeCU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -149,6 +149,7 @@ github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= From 9df707242c8ca77d572c1d47ece26f187a80a90b Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 25 Feb 2025 01:13:10 +0530 Subject: [PATCH 93/98] go mod update --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 814010481..dbc6eb85b 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.1 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.17.0-RC6 + github.com/0chain/gosdk v1.19.6 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 diff --git a/go.sum b/go.sum index bd8c07625..62e05127d 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v1.18.3 h1:42dYOv2KyMTSanuS67iDtfv+ErbSRqR8NJ3MG72MwaI= github.com/0chain/common v1.18.3/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.17.0-RC6 h1:NB4pfu0VbVWfQQEfnFqn5inaiQuyhSaUW7VwgH3LW8U= -github.com/0chain/gosdk v1.17.0-RC6/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg= +github.com/0chain/gosdk v1.19.6 h1:GfNdBBC0uQqaooT4Vh6B7FP1m76lyTagugizwfYYleQ= +github.com/0chain/gosdk v1.19.6/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw= github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f h1:H3Edk0y4Q8GEsPAbIsmn+WKiTC/uA4L/YuX0x/3Adz0= github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f/go.mod h1:aGBabvKYk/B6G6bEBI8j1cBaWY2is9sCii88Ly6VeCU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -149,7 +149,6 @@ github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= From 1e12d3a263a80d0db273e6bf46bceade48cef31a Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 25 Feb 2025 01:27:17 +0530 Subject: [PATCH 94/98] version fix --- docker.local/base.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index ae00982f2..f5d3dc6bf 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21-alpine3.18 as blobber_base +FROM golang:1.22-alpine3.18 as blobber_base LABEL zchain="blobber" From e54c557d3465b6340ef83298cef3be57b6d79007 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Wed, 26 Feb 2025 18:41:45 +0530 Subject: [PATCH 95/98] added logs --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 455d29519..36374892e 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -523,7 +523,7 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req if err != nil { return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+err.Error()) } - return nil, common.NewError("invalid_signature", "could not verify the allocation owner") + return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+": "+publicKey+": "+allocationId+": "+allocationTx+": "+clientSignV2) } var vm *writemarker.VersionMarker From 0784667c0482a3542c9b7ed252343e119817d7b9 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Wed, 19 Mar 2025 01:27:33 +0530 Subject: [PATCH 96/98] go mod update --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dbc6eb85b..93195d1f6 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( ) require ( - github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f + github.com/0chain/gosdk_common v0.0.0-20250318142402-b7bd1ebd4f66 github.com/lithammer/shortuuid/v3 v3.0.7 golang.org/x/sync v0.7.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc diff --git a/go.sum b/go.sum index 62e05127d..c052bf306 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= github.com/0chain/gosdk v1.19.6 h1:GfNdBBC0uQqaooT4Vh6B7FP1m76lyTagugizwfYYleQ= github.com/0chain/gosdk v1.19.6/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw= -github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f h1:H3Edk0y4Q8GEsPAbIsmn+WKiTC/uA4L/YuX0x/3Adz0= -github.com/0chain/gosdk_common v0.0.0-20250204070817-3c82b7ad6a9f/go.mod h1:aGBabvKYk/B6G6bEBI8j1cBaWY2is9sCii88Ly6VeCU= +github.com/0chain/gosdk_common v0.0.0-20250318142402-b7bd1ebd4f66 h1:lDzI8ZW/mdqfT0ySRnNke2c5KoqVh+FNXY8536yn0DA= +github.com/0chain/gosdk_common v0.0.0-20250318142402-b7bd1ebd4f66/go.mod h1:+FBmkG4JocGmD+8gyE4kyst15p+3xm3Nf+nrg4cu5V4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= From 1c0f09783a4f2d76e03b1f2420a455677ac54e11 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Wed, 19 Mar 2025 20:18:19 +0530 Subject: [PATCH 97/98] test --- code/go/0chain.net/blobbercore/handler/handler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index eeb943e8a..53545d428 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -311,21 +311,21 @@ func Authenticate0Box(handler common.ReqRespHandlerf) common.ReqRespHandlerf { signature := r.Header.Get("Zbox-Signature") if signature == "" { w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Invalid signature")) // nolint + w.Write([]byte("Invalid signature " + signature)) // nolint return } signatureScheme := zcncrypto.NewSignatureScheme(config.Configuration.SignatureScheme) if err := signatureScheme.SetPublicKey(common.PublicKey0box); err != nil { w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Invalid signature")) // nolint + w.Write([]byte("Invalid signature 2")) // nolint return } success, err := signatureScheme.Verify(signature, hex.EncodeToString([]byte(common.PublicKey0box))) if err != nil || !success { w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Invalid signature")) // nolint + w.Write([]byte("Invalid signature 3")) // nolint return } From 292412cdaa3a16f2352e2985213a02815113b578 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Thu, 20 Mar 2025 00:07:33 +0530 Subject: [PATCH 98/98] test logs --- code/go/0chain.net/blobbercore/handler/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 53545d428..9f478547e 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -325,7 +325,7 @@ func Authenticate0Box(handler common.ReqRespHandlerf) common.ReqRespHandlerf { success, err := signatureScheme.Verify(signature, hex.EncodeToString([]byte(common.PublicKey0box))) if err != nil || !success { w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Invalid signature 3")) // nolint + w.Write([]byte("Invalid signature 3" + common.PublicKey0box)) // nolint return }