From df53a00e8cd175434d0c7cb00f1fa9dad17b263c Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Mon, 16 Dec 2024 07:42:34 +0300 Subject: [PATCH 01/22] training v2 init --- authutils/auth_service.go | 36 +- blockchain/serviceMetadata.go | 44 +- escrow/control_service.go | 2 +- escrow/control_service_test.go | 1 - escrow/state_service.go | 1 - escrow/validation.go | 1 - go.mod | 28 +- go.sum | 32 ++ handler/grpc.go | 5 +- ipfsutils/compressed.go | 8 +- snetd/cmd/components.go | 16 +- snetd/cmd/serve.go | 2 +- training/mock.go | 141 +++++ training/service.go | 938 ++++++++++++++++++--------------- training/service_test.go | 12 +- training/storage.go | 50 +- training/training.proto | 111 ---- training/training_daemon.proto | 124 +++++ training/training_v2.proto | 119 +++++ training/util.go | 68 +++ utils/common.go | 14 - 21 files changed, 1106 insertions(+), 647 deletions(-) create mode 100644 training/mock.go delete mode 100644 training/training.proto create mode 100644 training/training_daemon.proto create mode 100644 training/training_v2.proto create mode 100644 training/util.go diff --git a/authutils/auth_service.go b/authutils/auth_service.go index f29b50ff..887935b2 100755 --- a/authutils/auth_service.go +++ b/authutils/auth_service.go @@ -8,11 +8,11 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/singnet/snet-daemon/v5/blockchain" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/singnet/snet-daemon/v5/blockchain" "go.uber.org/zap" ) @@ -25,6 +25,18 @@ const ( AllowedBlockChainDifference = 5 ) +func VerifySigner(message []byte, signature []byte, signer common.Address) error { + derivedSigner, err := GetSignerAddressFromMessage(message, signature) + if err != nil { + zap.L().Error(err.Error()) + return err + } + if err = VerifyAddress(*derivedSigner, signer); err != nil { + return err + } + return nil +} + func GetSignerAddressFromMessage(message, signature []byte) (signer *common.Address, err error) { messageFieldLog := zap.String("message", blockchain.BytesToBase64(message)) signatureFieldLog := zap.String("signature", blockchain.BytesToBase64(signature)) @@ -69,17 +81,17 @@ func GetSignerAddressFromMessage(message, signature []byte) (signer *common.Addr // VerifySigner Verify the signature done by given singer or not // returns nil if signer indeed sign the message and signature proves it, if not throws an error -func VerifySigner(message []byte, signature []byte, signer common.Address) error { - signerFromMessage, err := GetSignerAddressFromMessage(message, signature) - if err != nil { - zap.L().Error("error from getSignerAddressFromMessage", zap.Error(err)) - return err - } - if signerFromMessage.String() == signer.String() { - return nil - } - return fmt.Errorf("incorrect signer") -} +//func VerifySigner(message []byte, signature []byte, signer common.Address) error { +// signerFromMessage, err := GetSignerAddressFromMessage(message, signature) +// if err != nil { +// zap.L().Error("error from getSignerAddressFromMessage", zap.Error(err)) +// return err +// } +// if signerFromMessage.String() == signer.String() { +// return nil +// } +// return fmt.Errorf("incorrect signer") +//} // CompareWithLatestBlockNumber Check if the block number passed is not more +- 5 from the latest block number on chain func CompareWithLatestBlockNumber(blockNumberPassed *big.Int) error { diff --git a/blockchain/serviceMetadata.go b/blockchain/serviceMetadata.go index be5f7f6f..14d60e8f 100644 --- a/blockchain/serviceMetadata.go +++ b/blockchain/serviceMetadata.go @@ -1,16 +1,15 @@ package blockchain import ( - "context" "encoding/json" "fmt" + "github.com/bufbuild/protocompile/linker" "github.com/singnet/snet-daemon/v5/errs" "math/big" "os" "slices" "strings" - "github.com/bufbuild/protocompile" pproto "github.com/emicklei/proto" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" @@ -18,7 +17,6 @@ import ( "github.com/singnet/snet-daemon/v5/config" "github.com/singnet/snet-daemon/v5/ipfsutils" "go.uber.org/zap" - "google.golang.org/protobuf/reflect/protoreflect" ) /* @@ -144,9 +142,6 @@ import ( ] } */ -const ( - serviceProto = "service.proto" -) type ServiceMetadata struct { Version int `json:"version"` @@ -166,18 +161,23 @@ type ServiceMetadata struct { freeCallSignerAddress common.Address isfreeCallAllowed bool freeCallsAllowed int - DynamicPriceMethodMapping map[string]string `json:"dynamic_pricing"` - TrainingMethods []string `json:"training_methods"` - ProtoFile protoreflect.FileDescriptor `json:"-"` + DynamicPriceMethodMapping map[string]string `json:"dynamic_pricing"` + TrainingMethods []string `json:"training_methods"` + TrainingMetadata map[string]any `json:"training_metadata"` + ProtoDescriptors linker.Files `json:"-"` + ProtoFiles map[string]string `json:"-"` } + type Tiers struct { Tiers Tier `json:"tier"` } + type AddOns struct { DiscountInPercentage float64 `json:"discountInPercentage"` AddOnCostInAGIX int `json:"addOnCostInAGIX"` Name string `json:"name"` } + type TierRange struct { High int `json:"high"` DiscountInPercentage float64 `json:"DiscountInPercentage"` @@ -501,16 +501,16 @@ func setServiceProto(metaData *ServiceMetadata) (err error) { zap.L().Error("Error in retrieving file from filecoin/ipfs", zap.Error(err)) } - protoFiles, err := ipfsutils.ReadFilesCompressed(rawFile) + metaData.ProtoFiles, err = ipfsutils.ReadFilesCompressed(rawFile) if err != nil { return err } - if metaData.ServiceType == "http" && len(protoFiles) > 1 { + if metaData.ServiceType == "http" && len(metaData.ProtoFiles) > 1 { zap.L().Fatal("Currently daemon support only one proto file for HTTP services!") } - for _, file := range protoFiles { + for _, file := range metaData.ProtoFiles { zap.L().Debug("Protofile", zap.String("file", file)) // If Dynamic pricing is enabled, there will be mandatory checks on the service proto @@ -527,10 +527,6 @@ func setServiceProto(metaData *ServiceMetadata) (err error) { metaData.TrainingMethods = trainingMethodMap } } - - if metaData.ServiceType == "http" { - metaData.ProtoFile = getFileDescriptor(file) - } } return nil @@ -578,19 +574,3 @@ func buildDynamicPricingMethodsMap(serviceProto *pproto.Proto) (dynamicPricingMe } return } - -func getFileDescriptor(protoContent string) protoreflect.FileDescriptor { - - accessor := protocompile.SourceAccessorFromMap(map[string]string{ - serviceProto: protoContent, - }) - compiler := protocompile.Compiler{ - Resolver: &protocompile.SourceResolver{Accessor: accessor}, - SourceInfoMode: protocompile.SourceInfoStandard, - } - fds, err := compiler.Compile(context.Background(), serviceProto) - if err != nil || fds == nil { - zap.L().Fatal("failed to analyze protofile"+errs.ErrDescURL(errs.InvalidProto), zap.Error(err)) - } - return fds.FindFileByPath(serviceProto) -} diff --git a/escrow/control_service.go b/escrow/control_service.go index a8e0c7c7..53622a1d 100644 --- a/escrow/control_service.go +++ b/escrow/control_service.go @@ -392,7 +392,7 @@ func (service *ProviderControlService) removeClaimedPayments() error { if !ok || err != nil { return err } - //Compare the state of this payment in progress with what is available in block chain + //Compare the state of this payment in progress with what is available in blockchain if blockChainChannel.Nonce.Cmp(payment.ChannelNonce) > 0 { //if the Nonce on this block chain is higher than that of the Payment, //means that the payment has been completed , hence update the etcd state with this diff --git a/escrow/control_service_test.go b/escrow/control_service_test.go index b1468f7b..f3c495c7 100644 --- a/escrow/control_service_test.go +++ b/escrow/control_service_test.go @@ -164,7 +164,6 @@ func (suite *ControlServiceTestSuite) TestStartClaimForMultipleChannels() { assert.Nil(suite.T(), err) assert.NotNil(suite.T(), replyListInProgress.Payments[0].Signature) assert.NotNil(suite.T(), replyListInProgress.Payments[0].Signature) - } func (suite *ControlServiceTestSuite) TestProviderControlService_checkMpeAddress() { diff --git a/escrow/state_service.go b/escrow/state_service.go index a7bba73b..059ff50d 100644 --- a/escrow/state_service.go +++ b/escrow/state_service.go @@ -131,7 +131,6 @@ func (service *PaymentChannelStateService) GetChannelState(context context.Conte return nil, err } if !ok { - zap.L().Error("old payment is not found in storage, nevertheless local channel nonce is not equal to the blockchain one", zap.Any("ChannelID", channelID)) return nil, errors.New("channel has different nonce in local storage and blockchain and old payment is not found in storage") } diff --git a/escrow/validation.go b/escrow/validation.go index 311d86d4..fc2dfaa2 100644 --- a/escrow/validation.go +++ b/escrow/validation.go @@ -67,7 +67,6 @@ func (validator *FreeCallPaymentValidator) Validate(payment *FreeCallPayment) (e } return nil - } // ChannelPaymentValidator validates payment using payment channel state. diff --git a/go.mod b/go.mod index 5e12012d..79212fdc 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23 require ( github.com/OneOfOne/go-utils v0.0.0-20180319162427-6019ff89a94e github.com/bufbuild/protocompile v0.14.1 - github.com/emicklei/proto v1.13.2 + github.com/emicklei/proto v1.13.3 github.com/ethereum/go-ethereum v1.14.12 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang-jwt/jwt/v5 v5.2.1 @@ -16,7 +16,7 @@ require ( github.com/improbable-eng/grpc-web v0.15.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/kubo v0.32.1 - github.com/magiconair/properties v1.8.7 + github.com/magiconair/properties v1.8.9 github.com/pkg/errors v0.9.1 github.com/rs/cors v1.11.1 github.com/rs/xid v1.6.0 @@ -26,14 +26,14 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 go.etcd.io/etcd/client/v3 v3.5.17 go.etcd.io/etcd/server/v3 v3.5.17 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.29.0 - golang.org/x/net v0.31.0 + golang.org/x/crypto v0.31.0 + golang.org/x/net v0.32.0 golang.org/x/time v0.8.0 - google.golang.org/grpc v1.68.0 + google.golang.org/grpc v1.69.0 google.golang.org/protobuf v1.35.2 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -46,7 +46,7 @@ require ( github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.14.3 // indirect + github.com/bits-and-blooms/bitset v1.19.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/caddyserver/certmagic v0.21.4 // indirect @@ -59,7 +59,7 @@ require ( github.com/cockroachdb/pebble v1.1.2 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/consensys/bavard v0.1.22 // indirect + github.com/consensys/bavard v0.1.24 // indirect github.com/consensys/gnark-crypto v0.14.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -74,7 +74,7 @@ require ( github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ethereum/c-kzg-4844 v1.0.3 // indirect - github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect + github.com/ethereum/go-verkle v0.2.2 // indirect github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flynn/noise v1.1.0 // indirect @@ -106,7 +106,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect - github.com/holiman/uint256 v1.3.1 // indirect + github.com/holiman/uint256 v1.3.2 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -256,14 +256,14 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/mod v0.21.0 // indirect - golang.org/x/sync v0.9.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.26.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 29664a42..2d199857 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA= github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.19.1 h1:mv2yVhy96D2CuskLPXnc58oJNMs5PCWjAZuyYU0p12M= +github.com/bits-and-blooms/bitset v1.19.1/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -129,6 +131,8 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1: github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A= github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= +github.com/consensys/bavard v0.1.24 h1:Lfe+bjYbpaoT7K5JTFoMi5wo9V4REGLvQQbHmatoN2I= +github.com/consensys/bavard v0.1.24/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= @@ -187,6 +191,8 @@ github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/u github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY= github.com/emicklei/proto v1.13.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emicklei/proto v1.13.3 h1:h2u3JgIKSBWHI0jTFFAxiIzOQAe+bRGlQ/WlaIa49Uk= +github.com/emicklei/proto v1.13.3/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -200,6 +206,8 @@ github.com/ethereum/go-ethereum v1.14.12 h1:8hl57x77HSUo+cXExrURjU/w1VhL+ShCTJrT github.com/ethereum/go-ethereum v1.14.12/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= +github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= +github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -439,6 +447,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= @@ -653,6 +663,8 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= +github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= @@ -1016,6 +1028,8 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= @@ -1186,6 +1200,10 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1272,6 +1290,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1292,6 +1312,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1357,6 +1379,8 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -1377,6 +1401,8 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1477,6 +1503,8 @@ google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCe google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -1500,6 +1528,10 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/grpc v1.69.0 h1:quSiOM1GJPmPH5XtU+BCoVXcDVJJAzNcoyfC2cCjGkI= +google.golang.org/grpc v1.69.0/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/handler/grpc.go b/handler/grpc.go index d6a5fa34..bf2f97fe 100644 --- a/handler/grpc.go +++ b/handler/grpc.go @@ -298,7 +298,7 @@ func (g grpcHandler) grpcToHTTP(srv any, inStream grpc.ServerStream) error { } // convert proto msg to json - jsonBody, err := protoToJson(g.serviceMetaData.ProtoFile, f.Data, method) + jsonBody, err := protoToJson(g.serviceMetaData.ProtoDescriptors[0], f.Data, method) if err != nil { return status.Errorf(codes.Internal, "protoToJson error: %+v", errs.ErrDescURL(errs.InvalidProto)) } @@ -371,7 +371,7 @@ func (g grpcHandler) grpcToHTTP(srv any, inStream grpc.ServerStream) error { } zap.L().Debug("Response from HTTP service", zap.String("response", string(resp))) - protoMessage, errMarshal := jsonToProto(g.serviceMetaData.ProtoFile, resp, method) + protoMessage, errMarshal := jsonToProto(g.serviceMetaData.ProtoDescriptors[0], resp, method) if errMarshal != nil { return status.Errorf(codes.Internal, "jsonToProto error: %+v%v", errMarshal, errs.ErrDescURL(errs.InvalidProto)) } @@ -418,6 +418,7 @@ func jsonToProto(protoFile protoreflect.FileDescriptor, json []byte, methodName } func protoToJson(protoFile protoreflect.FileDescriptor, in []byte, methodName string) (json []byte, err error) { + if protoFile.Services().Len() == 0 { zap.L().Warn("service in proto not found") return []byte("error, invalid proto file"), errors.New("services in proto not found") diff --git a/ipfsutils/compressed.go b/ipfsutils/compressed.go index 43d79f23..e7dc89f2 100644 --- a/ipfsutils/compressed.go +++ b/ipfsutils/compressed.go @@ -11,10 +11,10 @@ import ( // ReadFilesCompressed - read all files which have been compressed, there can be more than one file // We need to start reading the proto files associated with the service. // proto files are compressed and stored as modelipfsHash -func ReadFilesCompressed(compressedFile []byte) (protofiles []string, err error) { +func ReadFilesCompressed(compressedFile []byte) (protos map[string]string, err error) { f := bytes.NewReader(compressedFile) tarReader := tar.NewReader(f) - protofiles = make([]string, 0) + protos = make(map[string]string) for { header, err := tarReader.Next() if err == io.EOF { @@ -36,7 +36,7 @@ func ReadFilesCompressed(compressedFile []byte) (protofiles []string, err error) zap.L().Error(err.Error()) return nil, err } - protofiles = append(protofiles, string(data)) + protos[name] = string(data) default: err = fmt.Errorf(fmt.Sprintf("%s : %c %s %s\n", "Unknown file Type ", @@ -48,5 +48,5 @@ func ReadFilesCompressed(compressedFile []byte) (protofiles []string, err error) return nil, err } } - return protofiles, nil + return protos, nil } diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index fe305273..10acf943 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -60,7 +60,7 @@ type Components struct { freeCallLockerStorage *storage.PrefixedAtomicStorage tokenManager token.Manager tokenService *escrow.TokenService - modelService training.ModelServer + trainingService training.DaemonServer modelUserStorage *training.ModelUserStorage modelStorage *training.ModelStorage } @@ -576,18 +576,18 @@ func (components *Components) ModelStorage() *training.ModelStorage { return components.modelStorage } -func (components *Components) ModelService() training.ModelServer { - if components.modelService != nil { - return components.modelService +func (components *Components) TrainingService() training.DaemonServer { + if components.trainingService != nil { + return components.trainingService } if !config.GetBool(config.BlockchainEnabledKey) { - components.modelService = &training.NoModelSupportService{} - return components.modelService + components.trainingService = training.NoTrainingService{} + return components.trainingService } - components.modelService = training.NewModelService(components.PaymentChannelService(), components.ServiceMetaData(), + components.trainingService = training.NewTrainingService(components.PaymentChannelService(), components.ServiceMetaData(), components.OrganizationMetaData(), components.ModelStorage(), components.ModelUserStorage()) - return components.modelService + return components.trainingService } func (components *Components) TokenManager() token.Manager { diff --git a/snetd/cmd/serve.go b/snetd/cmd/serve.go index e349a47b..7b25192d 100644 --- a/snetd/cmd/serve.go +++ b/snetd/cmd/serve.go @@ -215,7 +215,7 @@ func (d *daemon) start() { escrow.RegisterProviderControlServiceServer(d.grpcServer, d.components.ProviderControlService()) escrow.RegisterFreeCallStateServiceServer(d.grpcServer, d.components.FreeCallStateService()) escrow.RegisterTokenServiceServer(d.grpcServer, d.components.TokenService()) - training.RegisterModelServer(d.grpcServer, d.components.ModelService()) + training.RegisterDaemonServer(d.grpcServer, d.components.TrainingService()) grpc_health_v1.RegisterHealthServer(d.grpcServer, d.components.DaemonHeartBeat()) configuration_service.RegisterConfigurationServiceServer(d.grpcServer, d.components.ConfigurationService()) mux := cmux.New(d.lis) diff --git a/training/mock.go b/training/mock.go new file mode 100644 index 00000000..c8bfba95 --- /dev/null +++ b/training/mock.go @@ -0,0 +1,141 @@ +package training + +import ( + "context" + "fmt" + "google.golang.org/protobuf/types/known/emptypb" +) + +type NoModelSupportService struct { +} + +type NoTrainingService struct { +} + +func (n NoTrainingService) mustEmbedUnimplementedDaemonServer() { + panic("implement me") +} + +func (n NoTrainingService) CreateModel(ctx context.Context, request *NewModelRequest) (*ModelResponse, error) { + panic("implement me") +} + +func (n NoTrainingService) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { + panic("implement me") +} + +func (n NoTrainingService) UploadAndValidate(server Daemon_UploadAndValidateServer) error { + panic("implement me") +} + +func (n NoTrainingService) ValidateModel(ctx context.Context, request *AuthValidateRequest) (*StatusResponse, error) { + panic("implement me") +} + +func (n NoTrainingService) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { + panic("implement me") +} + +func (n NoTrainingService) TrainModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { + panic("implement me") +} + +func (n NoTrainingService) DeleteModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { + panic("implement me") +} + +func (n NoTrainingService) GetTrainingMetadata(ctx context.Context, empty *emptypb.Empty) (*TrainingMetadata, error) { + panic("implement me") +} + +func (n NoTrainingService) GetAllModels(ctx context.Context, request *AllModelsRequest) (*ModelsResponse, error) { + panic("implement me") +} + +func (n NoTrainingService) GetModel(ctx context.Context, request *CommonRequest) (*ModelResponse, error) { + panic("implement me") +} + +func (n NoTrainingService) UpdateModel(ctx context.Context, request *UpdateModelRequest) (*ModelResponse, error) { + panic("implement me") +} + +func (n NoTrainingService) GetMethodMetadata(ctx context.Context, request *MethodMetadataRequest) (*MethodMetadata, error) { + panic("implement me") +} + +func (service ModelService) CreateModel(ctx context.Context, model *NewModel) (*ModelID, error) { + panic("implement me") +} + +func (service ModelService) ValidateModelPrice(ctx context.Context, request *ValidateRequest) (*PriceInBaseUnit, error) { + panic("implement me") +} + +func (service ModelService) UploadAndValidate(server Model_UploadAndValidateServer) error { + panic("implement me") +} + +func (service ModelService) ValidateModel(ctx context.Context, request *ValidateRequest) (*StatusResponse, error) { + panic("implement me") +} + +func (service ModelService) TrainModelPrice(ctx context.Context, id *ModelID) (*PriceInBaseUnit, error) { + panic("implement me") +} + +func (service ModelService) TrainModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + panic("implement me") +} + +func (service ModelService) DeleteModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + panic("implement me") +} + +func (service ModelService) GetModelStatus(ctx context.Context, id *ModelID) (*StatusResponse, error) { + panic("implement me") +} + +func (service ModelService) mustEmbedUnimplementedModelServer() { + panic("implement me") +} + +func (n NoModelSupportService) CreateModel(ctx context.Context, model *NewModel) (*ModelID, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) ValidateModelPrice(ctx context.Context, request *ValidateRequest) (*PriceInBaseUnit, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) UploadAndValidate(server Model_UploadAndValidateServer) error { + return fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) ValidateModel(ctx context.Context, request *ValidateRequest) (*StatusResponse, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) TrainModelPrice(ctx context.Context, id *ModelID) (*PriceInBaseUnit, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) TrainModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) DeleteModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) GetModelStatus(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +} + +func (n NoModelSupportService) mustEmbedUnimplementedModelServer() { + panic("implement me") +} + +func (ds DaemonService) mustEmbedUnimplementedDaemonServer() { + panic("implement me") +} diff --git a/training/service.go b/training/service.go index 1f8930e4..5bf1088d 100644 --- a/training/service.go +++ b/training/service.go @@ -1,24 +1,30 @@ -//go:generate protoc -I . ./training.proto --go-grpc_out=. --go_out=. +//go:generate protoc -I . ./training_v2.proto --go-grpc_out=. --go_out=. +//go:generate protoc -I . ./training_daemon.proto --go-grpc_out=. --go_out=. + package training import ( - "bytes" + "context" "fmt" + "github.com/bufbuild/protocompile" + "github.com/bufbuild/protocompile/linker" + "github.com/singnet/snet-daemon/v5/blockchain" + "github.com/singnet/snet-daemon/v5/errs" "google.golang.org/grpc/credentials/insecure" - "math/big" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/structpb" + "maps" "net/url" "slices" "strings" "time" - "github.com/singnet/snet-daemon/v5/blockchain" + _ "embed" "github.com/singnet/snet-daemon/v5/config" "github.com/singnet/snet-daemon/v5/escrow" - "github.com/singnet/snet-daemon/v5/utils" - - "github.com/ethereum/go-ethereum/common/math" "go.uber.org/zap" - "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) @@ -27,8 +33,13 @@ const ( DateFormat = "02-01-2006" ) -type IService interface { -} +//go:embed training_v2.proto +var TrainingProtoEmbeded string + +//type IService interface { +//} + +// ModelService this is remote AI service provider type ModelService struct { serviceMetaData *blockchain.ServiceMetadata organizationMetaData *blockchain.OrganizationMetaData @@ -38,43 +49,114 @@ type ModelService struct { serviceUrl string } -func (service ModelService) mustEmbedUnimplementedModelServer() { - //TODO implement me +type DaemonService struct { + serviceMetaData *blockchain.ServiceMetadata + organizationMetaData *blockchain.OrganizationMetaData + channelService escrow.PaymentChannelService + storage *ModelStorage + userStorage *ModelUserStorage + serviceUrl string + trainingMetadata *TrainingMetadata + methodsMetadata map[string]*MethodMetadata +} + +func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest) (*ModelResponse, error) { + + if request == nil || request.Authorization == nil { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("invalid request, no authorization provided") + } + + if err := ds.verifySignature(request.Authorization); err != nil { + zap.L().Error("unable to create model", zap.Error(err)) + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("unable to create model: %v", err) + } + + if request.GetModel().GrpcServiceName == "" || request.GetModel().GrpcMethodName == "" { + zap.L().Error("invalid request, no grpc_method_name or grpc_service_name provided") + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("invalid request, no grpc_service_name or grpc_method_name provided") + } + + request.Model.ServiceId = config.GetString(config.ServiceId) + request.Model.OrganizationId = config.GetString(config.OrganizationId) + request.Model.GroupId = config.GetString(config.DaemonGroupName) + + // make a call to the client + // if the response is successful, store details in etcd + // send back the response to the client + conn, client, err := ds.getServiceClient() + deferConnection(conn) + if err != nil { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("error in invoking service for Model Training %v", err) + } + + responseModelID, errClient := client.CreateModel(c, request.Model) + if errClient != nil { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("error in invoking service for Model Training %v", errClient) + } + + if responseModelID == nil { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("error in invoking service for model training: service return empty response") + } + + if responseModelID.ModelId == "" { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("error in invoking service for model training: service return empty modelID") + } + + //store the details in etcd + zap.L().Info("Creating model based on response from CreateModel of training service") + + data, err := ds.createModelDetails(request, responseModelID) + if err != nil { + zap.L().Error(err.Error()) + return nil, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) + } + modelResponse := BuildModelResponse(data, Status_CREATED) + return modelResponse, err +} + +func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { panic("implement me") } -type NoModelSupportService struct { +func (ds *DaemonService) UploadAndValidate(server Daemon_UploadAndValidateServer) error { + panic("implement me") } -func (n NoModelSupportService) mustEmbedUnimplementedModelServer() { - //TODO implement me +func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidateRequest) (*StatusResponse, error) { panic("implement me") } -func (n NoModelSupportService) GetAllModels(c context.Context, request *AccessibleModelsRequest) (*AccessibleModelsResponse, error) { - return &AccessibleModelsResponse{}, - fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +func (ds *DaemonService) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { + panic("implement me") } -func (n NoModelSupportService) CreateModel(c context.Context, request *CreateModelRequest) (*ModelDetailsResponse, error) { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { + panic("implement me") } -func (n NoModelSupportService) UpdateModelAccess(c context.Context, request *UpdateModelRequest) (*ModelDetailsResponse, error) { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +func (ds *DaemonService) GetTrainingMetadata(ctx context.Context, empty *emptypb.Empty) (*TrainingMetadata, error) { + return ds.trainingMetadata, nil } -func (n NoModelSupportService) DeleteModel(c context.Context, request *UpdateModelRequest) (*ModelDetailsResponse, error) { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") +func (ds *DaemonService) UpdateModel(ctx context.Context, request *UpdateModelRequest) (*ModelResponse, error) { + panic("implement me") } -func (n NoModelSupportService) GetModelStatus(c context.Context, id *ModelDetailsRequest) (*ModelDetailsResponse, error) { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("service end point is not defined or is invalid for training , please contact the AI developer") +func (ds *DaemonService) GetMethodMetadata(ctx context.Context, request *MethodMetadataRequest) (*MethodMetadata, error) { + if request.GetModelId() != "" { + // TODO get from etcd + } + key := request.GrpcServiceName + request.GrpcMethodName + return ds.methodsMetadata[key], nil } + func deferConnection(conn *grpc.ClientConn) { defer func(conn *grpc.ClientConn) { err := conn.Close() @@ -83,6 +165,7 @@ func deferConnection(conn *grpc.ClientConn) { } }(conn) } + func getConnection(endpoint string) (conn *grpc.ClientConn) { options := grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(config.GetInt(config.MaxMessageSizeInMB)*1024*1024), @@ -108,28 +191,28 @@ func getConnection(endpoint string) (conn *grpc.ClientConn) { return } -func (service ModelService) getServiceClient() (conn *grpc.ClientConn, client ModelClient, err error) { - conn = getConnection(service.serviceUrl) +func (ds *DaemonService) getServiceClient() (conn *grpc.ClientConn, client ModelClient, err error) { + conn = getConnection(ds.serviceUrl) client = NewModelClient(conn) return } -func (service ModelService) createModelDetails(request *CreateModelRequest, response *ModelDetailsResponse) (data *ModelData, err error) { - key := service.getModelKeyToCreate(request, response) - data = service.getModelDataToCreate(request, response) +func (ds *DaemonService) createModelDetails(request *NewModelRequest, response *ModelID) (data *ModelData, err error) { + key := ds.buildModelKey(response.ModelId) + data = ds.getModelDataToCreate(request, response) //store the model details in etcd zap.L().Debug("createModelDetails", zap.Any("key", key)) - err = service.storage.Put(key, data) + err = ds.storage.Put(key, data) if err != nil { zap.L().Error("can't put model in etcd", zap.Error(err)) return } - //for every accessible address in the list , store the user address and all the model Ids associated with it + //for every accessible address in the list, store the user address and all the model Ids associated with it for _, address := range data.AuthorizedAddresses { userKey := getModelUserKey(key, address) - userData := service.getModelUserData(key, address) + userData := ds.getModelUserData(key, address) zap.L().Debug("createModelDetails", zap.Any("userKey", userKey)) - err = service.userStorage.Put(userKey, userData) + err = ds.userStorage.Put(userKey, userData) if err != nil { zap.L().Error("can't put in user storage", zap.Error(err)) return @@ -138,23 +221,24 @@ func (service ModelService) createModelDetails(request *CreateModelRequest, resp } return } + func getModelUserKey(key *ModelKey, address string) *ModelUserKey { return &ModelUserKey{ - OrganizationId: key.OrganizationId, - ServiceId: key.ServiceId, - GroupId: key.GroupId, - GRPCMethodName: key.GRPCMethodName, - GRPCServiceName: key.GRPCServiceName, - UserAddress: address, + OrganizationId: key.OrganizationId, + ServiceId: key.ServiceId, + GroupId: key.GroupId, + //GRPCMethodName: key.GRPCMethodName, + //GRPCServiceName: key.GRPCServiceName, + UserAddress: address, } } -func (service ModelService) getModelUserData(key *ModelKey, address string) *ModelUserData { +func (ds *DaemonService) getModelUserData(key *ModelKey, address string) *ModelUserData { //Check if there are any model Ids already associated with this user modelIds := make([]string, 0) userKey := getModelUserKey(key, address) zap.L().Debug("user model key is" + userKey.String()) - data, ok, err := service.userStorage.Get(userKey) + data, ok, err := ds.userStorage.Get(userKey) if ok && err == nil && data != nil { modelIds = data.ModelIds } @@ -163,23 +247,22 @@ func (service ModelService) getModelUserData(key *ModelKey, address string) *Mod } modelIds = append(modelIds, key.ModelId) return &ModelUserData{ - OrganizationId: key.OrganizationId, - ServiceId: key.ServiceId, - GroupId: key.GroupId, - GRPCMethodName: key.GRPCMethodName, - GRPCServiceName: key.GRPCServiceName, - UserAddress: address, - ModelIds: modelIds, + OrganizationId: key.OrganizationId, + ServiceId: key.ServiceId, + GroupId: key.GroupId, + //GRPCMethodName: key.GRPCMethodName, + //GRPCServiceName: key.GRPCServiceName, + UserAddress: address, + ModelIds: modelIds, } } -func (service ModelService) deleteUserModelDetails(key *ModelKey, data *ModelData) (err error) { - +func (ds DaemonService) deleteUserModelDetails(key *ModelKey, data *ModelData) (err error) { for _, address := range data.AuthorizedAddresses { userKey := getModelUserKey(key, address) - if data, ok, err := service.userStorage.Get(userKey); ok && err == nil && data != nil { + if data, ok, err := ds.userStorage.Get(userKey); ok && err == nil && data != nil { data.ModelIds = remove(data.ModelIds, key.ModelId) - err = service.userStorage.Put(userKey, data) + err = ds.userStorage.Put(userKey, data) if err != nil { zap.L().Error(err.Error()) } @@ -188,137 +271,96 @@ func (service ModelService) deleteUserModelDetails(key *ModelKey, data *ModelDat return } -func remove(s []string, r string) []string { - for i, v := range s { - if v == r { - return append(s[:i], s[i+1:]...) - } - } - return s -} - -func (service ModelService) deleteModelDetails(request *UpdateModelRequest) (data *ModelData, err error) { - key := service.getModelKeyToUpdate(request) +func (ds *DaemonService) deleteModelDetails(req *CommonRequest) (data *ModelData, err error) { + key := ds.getModelKeyToUpdate(req.ModelId) ok := false - data, ok, err = service.storage.Get(key) + data, ok, err = ds.storage.Get(key) if ok && err == nil { data.Status = Status_DELETED data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) - err = service.storage.Put(key, data) - err = service.deleteUserModelDetails(key, data) + err = ds.storage.Put(key, data) + err = ds.deleteUserModelDetails(key, data) } return } -func convertModelDataToBO(data *ModelData) (responseData *ModelDetails) { - responseData = &ModelDetails{ - ModelId: data.ModelId, - GrpcMethodName: data.GRPCMethodName, - GrpcServiceName: data.GRPCServiceName, - Description: data.Description, - IsPubliclyAccessible: data.IsPublic, - AddressList: data.AuthorizedAddresses, - TrainingDataLink: data.TrainingLink, - ModelName: data.ModelName, - OrganizationId: data.OrganizationId, - ServiceId: data.ServiceId, - GroupId: data.GroupId, - UpdatedDate: data.UpdatedDate, - Status: data.Status.String(), - } - return -} - -func (service ModelService) updateModelDetails(request *UpdateModelRequest, response *ModelDetailsResponse) (data *ModelData, err error) { - key := service.getModelKeyToUpdate(request) - oldAddresses := make([]string, 0) - var latestAddresses []string - // by default add the creator to the Authorized list of Address - if request.UpdateModelDetails.AddressList != nil || len(request.UpdateModelDetails.AddressList) > 0 { - latestAddresses = request.UpdateModelDetails.AddressList - } - latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) - if data, err = service.getModelDataForUpdate(request); err == nil && data != nil { - oldAddresses = data.AuthorizedAddresses - data.AuthorizedAddresses = latestAddresses - latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) - data.IsPublic = request.UpdateModelDetails.IsPubliclyAccessible - data.UpdatedByAddress = request.Authorization.SignerAddress - if response != nil { - data.Status = response.Status - } - data.ModelName = request.UpdateModelDetails.ModelName - data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) - data.Description = request.UpdateModelDetails.Description - - err = service.storage.Put(key, data) - if err != nil { - zap.L().Error("Error in putting data in user storage", zap.Error(err)) - } - //get the difference of all the addresses b/w old and new - updatedAddresses := difference(oldAddresses, latestAddresses) - for _, address := range updatedAddresses { - modelUserKey := getModelUserKey(key, address) - modelUserData := service.getModelUserData(key, address) - //if the address is present in the request but not in the old address , add it to the storage - if isValuePresent(address, request.UpdateModelDetails.AddressList) { - modelUserData.ModelIds = append(modelUserData.ModelIds, request.UpdateModelDetails.ModelId) - } else { // the address was present in the old data , but not in new , hence needs to be deleted - modelUserData.ModelIds = remove(modelUserData.ModelIds, request.UpdateModelDetails.ModelId) - } - err = service.userStorage.Put(modelUserKey, modelUserData) - if err != nil { - zap.L().Error("Error in putting data in storage", zap.Error(err)) - } - } +func convertModelDataToBO(data *ModelData) (responseData *ModelResponse) { + responseData = &ModelResponse{ + ModelId: data.ModelId, + GrpcMethodName: data.GRPCMethodName, + GrpcServiceName: data.GRPCServiceName, + Description: data.Description, + IsPublic: data.IsPublic, + AddressList: data.AuthorizedAddresses, + TrainingDataLink: data.TrainingLink, + Name: data.ModelName, + //OrganizationId: data.OrganizationId, + //ServiceId: data.ServiceId, + //GroupId: data.GroupId, + UpdatedDate: data.UpdatedDate, + Status: data.Status, } return } -func difference(oldAddresses []string, newAddresses []string) []string { - var diff []string - for i := 0; i < 2; i++ { - for _, s1 := range oldAddresses { - found := false - for _, s2 := range newAddresses { - if s1 == s2 { - found = true - break - } - } - // String not found. We add it to return slice - if !found { - diff = append(diff, s1) - } - } - // Swap the slices, only if it was the first loop - if i == 0 { - oldAddresses, newAddresses = newAddresses, oldAddresses - } - } - return diff -} - -func isValuePresent(value string, list []string) bool { - for _, v := range list { - if v == value { - return true - } - } - return false -} +//func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest, response *ModelDetailsResponse) (data *ModelData, err error) { +// key := ds.getModelKeyToUpdate(request.Mo) +// oldAddresses := make([]string, 0) +// var latestAddresses []string +// // by default add the creator to the Authorized list of Address +// if request.UpdateModelDetails.AddressList != nil || len(request.UpdateModelDetails.AddressList) > 0 { +// latestAddresses = request.UpdateModelDetails.AddressList +// } +// latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) +// if data, err = ds.getModelDataForUpdate(request); err == nil && data != nil { +// oldAddresses = data.AuthorizedAddresses +// data.AuthorizedAddresses = latestAddresses +// latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) +// data.IsPublic = request.UpdateModelDetails.IsPubliclyAccessible +// data.UpdatedByAddress = request.Authorization.SignerAddress +// if response != nil { +// data.Status = response.Status +// } +// data.ModelName = request.UpdateModelDetails.ModelName +// data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) +// data.Description = request.UpdateModelDetails.Description +// +// err = ds.storage.Put(key, data) +// if err != nil { +// zap.L().Error("Error in putting data in user storage", zap.Error(err)) +// } +// //get the difference of all the addresses b/w old and new +// updatedAddresses := difference(oldAddresses, latestAddresses) +// for _, address := range updatedAddresses { +// modelUserKey := getModelUserKey(key, address) +// modelUserData := service.getModelUserData(key, address) +// //if the address is present in the request but not in the old address , add it to the storage +// if isValuePresent(address, request.UpdateModelDetails.AddressList) { +// modelUserData.ModelIds = append(modelUserData.ModelIds, request.UpdateModelDetails.ModelId) +// } else { // the address was present in the old data , but not in new , hence needs to be deleted +// modelUserData.ModelIds = remove(modelUserData.ModelIds, request.UpdateModelDetails.ModelId) +// } +// err = ds.userStorage.Put(modelUserKey, modelUserData) +// if err != nil { +// zap.L().Error("Error in putting data in storage", zap.Error(err)) +// } +// } +// +// } +// return +//} // ensure only authorized use -func (service ModelService) verifySignerHasAccessToTheModel(serviceName string, methodName string, modelId string, address string) (err error) { +func (ds DaemonService) verifySignerHasAccessToTheModel(modelId string, address string) (err error) { key := &ModelUserKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: service.organizationMetaData.GetGroupIdString(), - GRPCMethodName: methodName, - GRPCServiceName: serviceName, - UserAddress: address, - } - data, ok, err := service.userStorage.Get(key) + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + //GRPCMethodName: methodName, + //GRPCServiceName: serviceName, + UserAddress: address, + } + data, ok, err := ds.userStorage.Get(key) if ok && err == nil { if !slices.Contains(data.ModelIds, modelId) { return fmt.Errorf("user %v, does not have access to model Id %v", address, modelId) @@ -327,20 +369,20 @@ func (service ModelService) verifySignerHasAccessToTheModel(serviceName string, return } -func (service ModelService) updateModelDetailsWithLatestStatus(request *ModelDetailsRequest, response *ModelDetailsResponse) (data *ModelData, err error) { +func (ds DaemonService) updateModelDetailsWithLatestStatus(request *CommonRequest, response *ModelResponse) (data *ModelData, err error) { key := &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: service.organizationMetaData.GetGroupIdString(), - GRPCMethodName: request.ModelDetails.GrpcMethodName, - GRPCServiceName: request.ModelDetails.GrpcServiceName, - ModelId: request.ModelDetails.ModelId, + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + //GRPCMethodName: request.ModelDetails.GrpcMethodName, + //GRPCServiceName: request.ModelDetails.GrpcServiceName, + ModelId: request.ModelId, } ok := false zap.L().Debug("updateModelDetailsWithLatestStatus: ", zap.Any("key", key)) - if data, ok, err = service.storage.Get(key); err == nil && ok { + if data, ok, err = ds.storage.Get(key); err == nil && ok { data.Status = response.Status - if err = service.storage.Put(key, data); err != nil { + if err = ds.storage.Put(key, data); err != nil { zap.L().Error("issue with retrieving model data from storage", zap.Error(err)) } } else { @@ -349,75 +391,75 @@ func (service ModelService) updateModelDetailsWithLatestStatus(request *ModelDet return } -func (service ModelService) getModelKeyToCreate(request *CreateModelRequest, response *ModelDetailsResponse) (key *ModelKey) { +func (ds *DaemonService) buildModelKey(modelID string) (key *ModelKey) { key = &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: service.organizationMetaData.GetGroupIdString(), - GRPCMethodName: request.ModelDetails.GrpcMethodName, - GRPCServiceName: request.ModelDetails.GrpcServiceName, - ModelId: response.ModelDetails.ModelId, + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + //GRPCMethodName: request.Model.GrpcMethodName, + //GRPCServiceName: request.Model.GrpcServiceName, + ModelId: modelID, } return } -func (service ModelService) getModelKeyToUpdate(request *UpdateModelRequest) (key *ModelKey) { +func (ds *DaemonService) getModelKeyToUpdate(modelID string) (key *ModelKey) { key = &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: service.organizationMetaData.GetGroupIdString(), - GRPCMethodName: request.UpdateModelDetails.GrpcMethodName, - GRPCServiceName: request.UpdateModelDetails.GrpcServiceName, - ModelId: request.UpdateModelDetails.ModelId, + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + //GRPCMethodName: request.UpdateModelDetails.GrpcMethodName, + //GRPCServiceName: request.UpdateModelDetails.GrpcServiceName, + ModelId: modelID, } return } -func (service ModelService) getModelDataForUpdate(request *UpdateModelRequest) (data *ModelData, err error) { - key := service.getModelKeyToUpdate(request) - ok := false - - if data, ok, err = service.storage.Get(key); err != nil || !ok { - zap.L().Warn("unable to retrieve model data from storage", zap.String("Model Id", key.ModelId), zap.Error(err)) - } - return -} +//func (ds *DaemonService) getModelDataForUpdate(request *UpdateModelRequest) (data *ModelData, err error) { +// key := ds.getModelKeyToUpdate(request) +// ok := false +// +// if data, ok, err = ds.storage.Get(key); err != nil || !ok { +// zap.L().Warn("unable to retrieve model data from storage", zap.String("Model Id", key.ModelId), zap.Error(err)) +// } +// return +//} -func (service ModelService) GetAllModels(c context.Context, request *AccessibleModelsRequest) (response *AccessibleModelsResponse, err error) { +func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (response *ModelsResponse, err error) { if request == nil || request.Authorization == nil { - return &AccessibleModelsResponse{}, + return &ModelsResponse{}, fmt.Errorf("Invalid request , no Authorization provided ") } - if err = service.verifySignature(request.Authorization); err != nil { - return &AccessibleModelsResponse{}, + if err = ds.verifySignature(request.Authorization); err != nil { + return &ModelsResponse{}, fmt.Errorf("Unable to access model, %v", err) } - if request.GetGrpcMethodName() == "" || request.GetGrpcServiceName() == "" { - return &AccessibleModelsResponse{}, - fmt.Errorf("Invalid request, no GrpcMethodName or GrpcServiceName provided") - } + //if request.GetGrpcMethodName() == "" || request.GetGrpcServiceName() == "" { + // return &AccessibleModelsResponse{}, + // fmt.Errorf("Invalid request, no GrpcMethodName or GrpcServiceName provided") + //} key := &ModelUserKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: service.organizationMetaData.GetGroupIdString(), - GRPCMethodName: request.GrpcMethodName, - GRPCServiceName: request.GrpcServiceName, - UserAddress: request.Authorization.SignerAddress, + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + //GRPCMethodName: request.GrpcMethodName, + //GRPCServiceName: request.GrpcServiceName, + UserAddress: request.Authorization.SignerAddress, } - modelDetailsArray := make([]*ModelDetails, 0) - if data, ok, err := service.userStorage.Get(key); data != nil && ok && err == nil { + modelDetailsArray := make([]*ModelResponse, 0) + if data, ok, err := ds.userStorage.Get(key); data != nil && ok && err == nil { for _, modelId := range data.ModelIds { modelKey := &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: service.organizationMetaData.GetGroupIdString(), - GRPCMethodName: request.GrpcMethodName, - GRPCServiceName: request.GrpcServiceName, - ModelId: modelId, + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + //GRPCMethodName: request.GrpcMethodName, + //GRPCServiceName: request.GrpcServiceName, + ModelId: modelId, } - if modelData, modelOk, modelErr := service.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { + if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { boModel := convertModelDataToBO(modelData) modelDetailsArray = append(modelDetailsArray, boModel) } @@ -425,34 +467,33 @@ func (service ModelService) GetAllModels(c context.Context, request *AccessibleM } for _, model := range modelDetailsArray { - zap.L().Debug("Model", zap.String("Name", model.ModelName)) + zap.L().Debug("Model", zap.String("Name", model.Name)) } - response = &AccessibleModelsResponse{ + response = &ModelsResponse{ ListOfModels: modelDetailsArray, } return } -func (service ModelService) getModelDataToCreate(request *CreateModelRequest, response *ModelDetailsResponse) (data *ModelData) { - +func (ds DaemonService) getModelDataToCreate(request *NewModelRequest, response *ModelID) (data *ModelData) { data = &ModelData{ - Status: response.Status, - GRPCServiceName: request.ModelDetails.GrpcServiceName, - GRPCMethodName: request.ModelDetails.GrpcMethodName, + Status: Status_CREATED, + GRPCServiceName: request.Model.GrpcServiceName, + GRPCMethodName: request.Model.GrpcMethodName, CreatedByAddress: request.Authorization.SignerAddress, UpdatedByAddress: request.Authorization.SignerAddress, - AuthorizedAddresses: request.ModelDetails.AddressList, - Description: request.ModelDetails.Description, - ModelName: request.ModelDetails.ModelName, - TrainingLink: request.ModelDetails.TrainingDataLink, - IsPublic: request.ModelDetails.IsPubliclyAccessible, - IsDefault: false, - ModelId: response.ModelDetails.ModelId, - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: service.organizationMetaData.GetGroupIdString(), - UpdatedDate: fmt.Sprintf("%v", time.Now().Format(DateFormat)), + AuthorizedAddresses: request.Model.AddressList, + Description: request.Model.Description, + ModelName: request.Model.Name, + //TrainingLink: + IsPublic: request.Model.IsPublic, + IsDefault: false, + ModelId: response.ModelId, + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + UpdatedDate: fmt.Sprintf("%v", time.Now().Format(DateFormat)), } //by default add the creator to the Authorized list of Address if data.AuthorizedAddresses == nil { @@ -462,214 +503,146 @@ func (service ModelService) getModelDataToCreate(request *CreateModelRequest, re return } -func (service ModelService) CreateModel(c context.Context, request *CreateModelRequest) (response *ModelDetailsResponse, - err error) { - - // verify the request - if request == nil || request.Authorization == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no Authorization provided , %v", err) - } - if err = service.verifySignature(request.Authorization); err != nil { - zap.L().Error(err.Error()) - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("unable to create Model: %v", err) - } - if request.GetModelDetails().GrpcServiceName == "" || request.GetModelDetails().GrpcMethodName == "" { - zap.L().Error("Error in getting grpc service name", zap.Error(err)) - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no GrpcServiceName or GrpcMethodName provided , %v", err) - } - - // make a call to the client - // if the response is successful, store details in etcd - // send back the response to the client - conn, client, err := service.getServiceClient() - if err != nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("error in invoking service for Model Training %v", err) - } - - response, err = client.CreateModel(c, request) - if err != nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("error in invoking service for Model Training %v", err) +func BuildModelResponse(data *ModelData, status Status) *ModelResponse { + return &ModelResponse{ + Status: status, + ModelId: data.ModelId, + GrpcMethodName: data.GRPCMethodName, + GrpcServiceName: data.GRPCServiceName, + Description: data.Description, + IsPublic: data.IsPublic, + AddressList: data.AuthorizedAddresses, + TrainingDataLink: data.TrainingLink, + Name: data.ModelName, + //OrganizationId: config.GetString(config.OrganizationId), + //ServiceId: config.GetString(config.ServiceId), + //GroupId: data.GroupId, + //Status: status.String(), + UpdatedDate: data.UpdatedDate, + } +} + +//func (ds *DaemonService) UpdateModelAccess(c context.Context, request *UpdateModelRequest) (response *ModelDetailsResponse, +// err error) { +// if request == nil || request.Authorization == nil { +// return &ModelDetailsResponse{Status: Status_ERRORED}, +// fmt.Errorf(" Invalid request , no Authorization provided , %v", err) +// } +// if err = ds.verifySignature(request.Authorization); err != nil { +// return &ModelDetailsResponse{Status: Status_ERRORED}, +// fmt.Errorf(" Unable to access model , %v", err) +// } +// if err = service.verifySignerHasAccessToTheModel(request.UpdateModelDetails.GrpcServiceName, +// request.UpdateModelDetails.GrpcMethodName, request.UpdateModelDetails.ModelId, request.Authorization.SignerAddress); err != nil { +// return &ModelDetailsResponse{}, +// fmt.Errorf(" Unable to access model , %v", err) +// } +// if request.UpdateModelDetails == nil { +// return &ModelDetailsResponse{Status: Status_ERRORED}, +// fmt.Errorf(" Invalid request , no UpdateModelDetails provided , %v", err) +// } +// +// zap.L().Info("Updating model based on response from UpdateModel") +// if data, err := service.updateModelDetails(request, response); err == nil && data != nil { +// response = BuildModelResponse(data, data.Status) +// +// } else { +// return response, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) +// } +// +// return +//} + +func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*StatusResponse, error) { + + if req == nil || req.Authorization == nil { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("invalid request, no Authorization provided") + } + + if req.ModelId == "" { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("invalid request: ModelId is empty") + } + + if err := ds.verifySignature(req.Authorization); err != nil { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("unable to access model , %v", err) } - if response.ModelDetails == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("error in invoking service for Model Training: service return empty ModelDetails") + if err := ds.verifySignerHasAccessToTheModel(req.ModelId, req.Authorization.SignerAddress); err != nil { + return &StatusResponse{}, + fmt.Errorf("unable to access model , %v", err) } - //store the details in etcd - zap.L().Info("Creating model based on response from CreateModel of training service") - - data, err := service.createModelDetails(request, response) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + conn, client, err := ds.getServiceClient() if err != nil { - zap.L().Error(err.Error()) - return response, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) + return &StatusResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for Model Training") } - response = BuildModelResponseFrom(data, response.Status) deferConnection(conn) - return -} - -func BuildModelResponseFrom(data *ModelData, status Status) *ModelDetailsResponse { - return &ModelDetailsResponse{ - Status: status, - ModelDetails: &ModelDetails{ - ModelId: data.ModelId, - GrpcMethodName: data.GRPCMethodName, - GrpcServiceName: data.GRPCServiceName, - Description: data.Description, - IsPubliclyAccessible: data.IsPublic, - AddressList: data.AuthorizedAddresses, - TrainingDataLink: data.TrainingLink, - ModelName: data.ModelName, - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: data.GroupId, - Status: status.String(), - UpdatedDate: data.UpdatedDate, - }, - } -} -func (service ModelService) UpdateModelAccess(c context.Context, request *UpdateModelRequest) (response *ModelDetailsResponse, - err error) { - if request == nil || request.Authorization == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Invalid request , no Authorization provided , %v", err) - } - if err = service.verifySignature(request.Authorization); err != nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Unable to access model , %v", err) - } - if err = service.verifySignerHasAccessToTheModel(request.UpdateModelDetails.GrpcServiceName, - request.UpdateModelDetails.GrpcMethodName, request.UpdateModelDetails.ModelId, request.Authorization.SignerAddress); err != nil { - return &ModelDetailsResponse{}, - fmt.Errorf(" Unable to access model , %v", err) + response, err := client.DeleteModel(ctx, &ModelID{ModelId: req.ModelId}) + if response == nil || err != nil { + zap.L().Error("error in invoking DeleteModel, service-provider should realize it", zap.Error(err)) + return &StatusResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking DeleteModel, service-provider should realize it") } - if request.UpdateModelDetails == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Invalid request , no UpdateModelDetails provided , %v", err) + data, err := ds.deleteModelDetails(req) + if err == nil && data != nil { + zap.L().Error("issue with deleting ModelId in storage", zap.Error(err)) + return response, fmt.Errorf("issue with deleting Model Id in Storage %v", err) } - - zap.L().Info("Updating model based on response from UpdateModel") - if data, err := service.updateModelDetails(request, response); err == nil && data != nil { - response = BuildModelResponseFrom(data, data.Status) - - } else { - return response, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) - } - - return + //responseData := BuildModelResponse(data, response.Status) + return &StatusResponse{Status: Status_DELETED}, err } -func (service ModelService) DeleteModel(c context.Context, request *UpdateModelRequest) (response *ModelDetailsResponse, +func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (response *ModelResponse, err error) { if request == nil || request.Authorization == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Invalid request , no Authorization provided , %v", err) - } - if err = service.verifySignature(request.Authorization); err != nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Unable to access model , %v", err) - } - if err = service.verifySignerHasAccessToTheModel(request.UpdateModelDetails.GrpcServiceName, - request.UpdateModelDetails.GrpcMethodName, request.UpdateModelDetails.ModelId, request.Authorization.SignerAddress); err != nil { - return &ModelDetailsResponse{}, - fmt.Errorf(" Unable to access model , %v", err) - } - if request.UpdateModelDetails == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Invalid request: UpdateModelDetails are empty") - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - if conn, client, err := service.getServiceClient(); err == nil { - response, err = client.DeleteModel(ctx, request) - if response == nil || err != nil { - zap.L().Error("error in invoking DeleteModel, service-provider should realize it", zap.Error(err)) - return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking DeleteModel, service-provider should realize it") - } - if data, err := service.deleteModelDetails(request); err == nil && data != nil { - response = BuildModelResponseFrom(data, response.Status) - } else { - zap.L().Error("issue with deleting ModelId in storage", zap.Error(err)) - return response, fmt.Errorf("issue with deleting Model Id in Storage %v", err) - } - deferConnection(conn) - } else { - return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for Model Training") - } - - return -} - -func (service ModelService) GetModelStatus(c context.Context, request *ModelDetailsRequest) (response *ModelDetailsResponse, - err error) { - if request == nil || request.Authorization == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, + return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("invalid request, no Authorization provided , %v", err) } - if err = service.verifySignature(request.Authorization); err != nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, + if err = ds.verifySignature(request.Authorization); err != nil { + return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("unable to access model , %v", err) } - if err = service.verifySignerHasAccessToTheModel(request.ModelDetails.GrpcServiceName, - request.ModelDetails.GrpcMethodName, request.ModelDetails.ModelId, request.Authorization.SignerAddress); err != nil { - return &ModelDetailsResponse{}, + if err = ds.verifySignerHasAccessToTheModel(request.ModelId, request.Authorization.SignerAddress); err != nil { + return &ModelResponse{}, fmt.Errorf("unable to access model , %v", err) } - if request.ModelDetails == nil { - return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request: ModelDetails can't be empty") + if request.ModelId == "" { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("invalid request: ModelId can't be empty") } ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() - if conn, client, err := service.getServiceClient(); err == nil { - response, err = client.GetModelStatus(ctx, request) - if response == nil || err != nil { + if conn, client, err := ds.getServiceClient(); err == nil { + responseStatus, err := client.GetModelStatus(ctx, &ModelID{ModelId: request.ModelId}) + if responseStatus == nil || err != nil { zap.L().Error("error in invoking GetModelStatus, service-provider should realize it", zap.Error(err)) - return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking GetModelStatus, service-provider should realize it") + return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking GetModelStatus, service-provider should realize it") } zap.L().Info("[GetModelStatus] response from service-provider", zap.Any("response", response)) zap.L().Info("[GetModelStatus] updating model status based on response from UpdateModel") - data, err := service.updateModelDetailsWithLatestStatus(request, response) + data, err := ds.updateModelDetailsWithLatestStatus(request, response) zap.L().Debug("[GetModelStatus] data that be returned to client", zap.Any("data", data)) if err == nil && data != nil { - response = BuildModelResponseFrom(data, response.Status) + response = BuildModelResponse(data, responseStatus.Status) } else { - zap.L().Error("[GetModelStatus] BuildModelResponseFrom error", zap.Error(err)) + zap.L().Error("[GetModelStatus] BuildModelResponse error", zap.Error(err)) return response, fmt.Errorf("[GetModelStatus] issue with storing Model Id in the Daemon Storage %v", err) } deferConnection(conn) } else { - return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("[GetModelStatus] error in invoking service for Model Training") + return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("[GetModelStatus] error in invoking service for Model Training") } return } -// message used to sign is of the form ("__create_model", mpe_address, current_block_number) -func (service *ModelService) verifySignature(request *AuthorizationDetails) error { - return utils.VerifySigner(service.getMessageBytes(request.Message, request), - request.GetSignature(), utils.ToChecksumAddress(request.SignerAddress)) -} - -// "user passed message ", user_address, current_block_number -func (service *ModelService) getMessageBytes(prefixMessage string, request *AuthorizationDetails) []byte { - userAddress := utils.ToChecksumAddress(request.SignerAddress) - message := bytes.Join([][]byte{ - []byte(prefixMessage), - userAddress.Bytes(), - math.U256Bytes(big.NewInt(int64(request.CurrentBlock))), - }, nil) - return message -} - +// NewModelService AI service prodiver +// TODO maybe remove func NewModelService(channelService escrow.PaymentChannelService, serMetaData *blockchain.ServiceMetadata, orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage) ModelServer { serviceURL := config.GetString(config.ModelMaintenanceEndPoint) @@ -686,3 +659,136 @@ func NewModelService(channelService escrow.PaymentChannelService, serMetaData *b return &NoModelSupportService{} } } + +// NewTrainingService daemon self server +func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData *blockchain.ServiceMetadata, + orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage) DaemonServer { + + linkerFiles := getFileDescriptors(serMetaData.ProtoFiles) + serMetaData.ProtoDescriptors = linkerFiles + methodsMD, trainMD, err := parseTrainingMetadata(linkerFiles) + if err != nil { + // TODO + return nil + } + + serviceURL := config.GetString(config.ModelMaintenanceEndPoint) + if config.IsValidUrl(serviceURL) && config.GetBool(config.BlockchainEnabledKey) { + return &DaemonService{ + channelService: channelService, + serviceMetaData: serMetaData, + organizationMetaData: orgMetadata, + storage: storage, + userStorage: userStorage, + serviceUrl: serviceURL, + trainingMetadata: &trainMD, + methodsMetadata: methodsMD, + } + } else { + return &NoTrainingService{} + } +} + +// parseTrainingMetadata TODO add comment +func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMetadata, trainingMD TrainingMetadata, err error) { + methodsMD = make(map[string]*MethodMetadata) + trainingMD.TrainingMethods = make(map[string]*structpb.ListValue) + + for _, protoFile := range protos { + for servicesCounter := 0; servicesCounter < protoFile.Services().Len(); servicesCounter++ { + protoService := protoFile.Services().Get(servicesCounter) + if protoService == nil { + continue + } + for methodsCounter := 0; methodsCounter < protoService.Methods().Len(); methodsCounter++ { + method := protoFile.Services().Get(servicesCounter).Methods().Get(methodsCounter) + if method == nil { + continue + } + inputFields := method.Input().Fields() + if inputFields == nil { + continue + } + for fieldsCounter := 0; fieldsCounter < inputFields.Len(); fieldsCounter++ { + if inputFields.Get(fieldsCounter).Message() != nil { + // if the method accepts modelId, then we consider it as training + if inputFields.Get(fieldsCounter).Message().FullName() == "trainingV2.ModelID" { + // init array if nil + if trainingMD.TrainingMethods[string(protoService.Name())] == nil { + trainingMD.TrainingMethods[string(protoService.Name())], _ = structpb.NewList(nil) + } + value := structpb.NewStringValue(string(method.Name())) + trainingMD.TrainingMethods[string(protoService.Name())].Values = append(trainingMD.TrainingMethods[string(protoService.Name())].Values, value) + } + } + } + + methodOptions, ok := method.Options().(*descriptorpb.MethodOptions) + if ok && methodOptions != nil { + key := string(protoService.Name() + method.Name()) + methodsMD[key] = &MethodMetadata{} + if proto.HasExtension(methodOptions, E_DatasetDescription) { + if datasetDesc, ok := proto.GetExtension(methodOptions, E_DatasetDescription).(string); ok { + methodsMD[key].DatasetDescription = datasetDesc + } + } + if proto.HasExtension(methodOptions, E_DatasetType) { + if datasetType, ok := proto.GetExtension(methodOptions, E_DatasetType).(string); ok { + methodsMD[key].DatasetType = datasetType + } + } + if proto.HasExtension(methodOptions, E_DatasetFilesType) { + if datasetDesc, ok := proto.GetExtension(methodOptions, E_DatasetFilesType).(string); ok { + methodsMD[key].DatasetFilesType = datasetDesc + } + } + if proto.HasExtension(methodOptions, E_MaxModelsPerUser) { + if datasetDesc, ok := proto.GetExtension(methodOptions, E_MaxModelsPerUser).(uint64); ok { + methodsMD[key].MaxModelsPerUser = datasetDesc + } + } + if proto.HasExtension(methodOptions, E_DefaultModelId) { + if defaultModelId, ok := proto.GetExtension(methodOptions, E_DefaultModelId).(string); ok { + methodsMD[key].DefaultModelId = defaultModelId + } + } + if proto.HasExtension(methodOptions, E_DatasetMaxSizeSingleFileMb) { + if d, ok := proto.GetExtension(methodOptions, E_DatasetMaxSizeSingleFileMb).(uint64); ok { + methodsMD[key].DatasetMaxSizeSingleFileMb = d + } + } + if proto.HasExtension(methodOptions, E_DatasetMaxCountFiles) { + if maxCountFiles, ok := proto.GetExtension(methodOptions, E_DatasetMaxCountFiles).(uint64); ok { + methodsMD[key].DatasetMaxCountFiles = maxCountFiles + } + } + if proto.HasExtension(methodOptions, E_DatasetMaxSizeMb) { + if datasetMaxSizeMb, ok := proto.GetExtension(methodOptions, E_DatasetMaxSizeMb).(uint64); ok { + methodsMD[key].DatasetMaxSizeMb = datasetMaxSizeMb + } + } + if methodsMD[key].DefaultModelId != "" { + zap.L().Debug("training metadata", zap.String("method", string(method.Name())), zap.String("key", key), zap.Any("metadata", methodsMD[key])) + } + } + } + } + } + zap.L().Debug("training methods", zap.Any("methods", trainingMD.TrainingMethods)) + return +} + +func getFileDescriptors(protoFiles map[string]string) linker.Files { + protoFiles["training_v2.proto"] = TrainingProtoEmbeded + accessor := protocompile.SourceAccessorFromMap(protoFiles) + r := protocompile.WithStandardImports(&protocompile.SourceResolver{Accessor: accessor}) + compiler := protocompile.Compiler{ + Resolver: r, + SourceInfoMode: protocompile.SourceInfoStandard, + } + fds, err := compiler.Compile(context.Background(), slices.Collect(maps.Keys(protoFiles))...) + if err != nil || fds == nil { + zap.L().Fatal("failed to analyze protofile"+errs.ErrDescURL(errs.InvalidProto), zap.Error(err)) + } + return fds +} diff --git a/training/service_test.go b/training/service_test.go index fdc611f1..0d20153d 100644 --- a/training/service_test.go +++ b/training/service_test.go @@ -42,6 +42,7 @@ type ModelServiceTestSuite struct { func TestModelServiceTestSuite(t *testing.T) { suite.Run(t, new(ModelServiceTestSuite)) } + func (suite *ModelServiceTestSuite) getGRPCServerAndServe() { config.Vip().Set(config.ModelMaintenanceEndPoint, "http://localhost:2222") config.Vip().Set(config.ModelTrainingEndpoint, "http://localhost:2222") @@ -61,6 +62,7 @@ func (suite *ModelServiceTestSuite) getGRPCServerAndServe() { _ = <-ch } + func (suite *ModelServiceTestSuite) SetupSuite() { suite.serviceNotImplemented = NewModelService(nil, nil, nil, nil, nil) config.Vip().Set(config.ModelMaintenanceEndPoint, "localhost:2222") @@ -74,7 +76,7 @@ func (suite *ModelServiceTestSuite) SetupSuite() { orgMetaData, _ := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) serviceMetaData, _ := blockchain.InitServiceMetaDataFromJson([]byte(testJsonData)) - suite.service = NewModelService(nil, serviceMetaData, orgMetaData, + suite.service = NewTrainingService(nil, serviceMetaData, orgMetaData, NewModelStorage(storage.NewMemStorage()), NewUerModelStorage(storage.NewMemStorage())) suite.senderPvtKy, _ = crypto.GenerateKey() suite.senderAddress = crypto.PubkeyToAddress(suite.senderPvtKy.PublicKey) @@ -125,10 +127,10 @@ func (m MockServiceModelGRPCImpl) GetModelStatus(context context.Context, reques }}, nil } -func (m MockServiceModelGRPCImpl) GetAllModels(context context.Context, request *AccessibleModelsRequest) (*AccessibleModelsResponse, error) { - //Ideally client should take a list of all models and update the status of each and send back a response - return &AccessibleModelsResponse{}, nil -} +//func (m MockServiceModelGRPCImpl) GetAllModels(context context.Context, request *AccessibleModelsRequest) (*AccessibleModelsResponse, error) { +// //Ideally client should take a list of all models and update the status of each and send back a response +// return &AccessibleModelsResponse{}, nil +//} func (suite *ModelServiceTestSuite) TearDownSuite() { suite.server.GracefulStop() diff --git a/training/storage.go b/training/storage.go index 1a0104d9..dde4428b 100644 --- a/training/storage.go +++ b/training/storage.go @@ -10,6 +10,7 @@ import ( type ModelStorage struct { delegate storage.TypedAtomicStorage } + type ModelUserStorage struct { delegate storage.TypedAtomicStorage } @@ -33,49 +34,49 @@ func NewModelStorage(atomicStorage storage.AtomicStorage) *ModelStorage { } type ModelUserKey struct { - OrganizationId string - ServiceId string - GroupId string - GRPCMethodName string - GRPCServiceName string - UserAddress string + OrganizationId string + ServiceId string + GroupId string + //GRPCMethodName string + //GRPCServiceName string + UserAddress string } func (key *ModelUserKey) String() string { - return fmt.Sprintf("{ID:%v|%v|%v|%v|%v|%v}", key.OrganizationId, - key.ServiceId, key.GroupId, key.GRPCServiceName, key.GRPCMethodName, key.UserAddress) + return fmt.Sprintf("{ID:%v|%v|%v|%v}", key.OrganizationId, + key.ServiceId, key.GroupId, key.UserAddress) } // ModelUserData maintain the list of all modelIds for a given user address type ModelUserData struct { ModelIds []string //the below are only for display purposes - OrganizationId string - ServiceId string - GroupId string - GRPCMethodName string - GRPCServiceName string - UserAddress string + OrganizationId string + ServiceId string + GroupId string + //GRPCMethodName string + //GRPCServiceName string + UserAddress string } func (data *ModelUserData) String() string { - return fmt.Sprintf("{DATA:%v|%v|%v|%v|%v|%v|%v}", + return fmt.Sprintf("{DATA:%v|%v|%v|%v|%v}", data.OrganizationId, - data.ServiceId, data.GroupId, data.GRPCMethodName, data.GRPCServiceName, data.UserAddress, data.ModelIds) + data.ServiceId, data.GroupId, data.UserAddress, data.ModelIds) } type ModelKey struct { - OrganizationId string - ServiceId string - GroupId string - GRPCMethodName string - GRPCServiceName string - ModelId string + OrganizationId string + ServiceId string + GroupId string + //GRPCMethodName string + //GRPCServiceName string + ModelId string } func (key *ModelKey) String() string { - return fmt.Sprintf("{ID:%v|%v|%v|%v|%v|%v}", key.OrganizationId, - key.ServiceId, key.GroupId, key.GRPCServiceName, key.GRPCMethodName, key.ModelId) + return fmt.Sprintf("{ID:%v|%v|%v|%v}", key.OrganizationId, + key.ServiceId, key.GroupId, key.ModelId) } func (data *ModelData) String() string { @@ -108,6 +109,7 @@ func serializeModelKey(key any) (serialized string, err error) { myKey := key.(*ModelKey) return myKey.String(), nil } + func (storage *ModelStorage) Get(key *ModelKey) (state *ModelData, ok bool, err error) { value, ok, err := storage.delegate.Get(key) if err != nil || !ok { diff --git a/training/training.proto b/training/training.proto deleted file mode 100644 index 892f51d4..00000000 --- a/training/training.proto +++ /dev/null @@ -1,111 +0,0 @@ -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -package training; -option go_package = "../training"; -//Please note that the AI developers need to provide a server implementation of the gprc server of this proto. -message ModelDetails { - //This Id will be generated when you invoke the create_model method and hence doesnt need to be filled when you - //invoke the create model - string model_id = 1; - //define the training method name - string grpc_method_name = 2; - //define the grpc service name , under which the method is defined - string grpc_service_name = 3; - string description = 4; - - string status = 6; - string updated_date = 7; - //List of all the addresses that will have access to this model - repeated string address_list = 8; - // this is optional - string training_data_link = 9; - string model_name = 10; - - - string organization_id = 11; - string service_id = 12 ; - string group_id = 13; - - //set this to true if you want your model to be used by other AI consumers - bool is_publicly_accessible = 14; -} - -message AuthorizationDetails { - uint64 current_block = 1; - //Signer can fill in any message here - string message = 2; - //signature of the following message: - //("user specified message", user_address, current_block_number) - bytes signature = 3; - string signer_address = 4; - -} - -enum Status { - CREATED = 0; - IN_PROGRESS = 1; - ERRORED = 2; - COMPLETED = 3; - DELETED = 4; -} - -message CreateModelRequest { - AuthorizationDetails authorization = 1; - ModelDetails model_details = 2; -} - -//the signer address will get to know all the models associated with this address. -message AccessibleModelsRequest { - string grpc_method_name = 1; - string grpc_service_name = 2; - AuthorizationDetails authorization = 3; -} - -message AccessibleModelsResponse { - repeated ModelDetails list_of_models = 1; -} - -message ModelDetailsRequest { - ModelDetails model_details = 1 ; - AuthorizationDetails authorization = 2; -} - -//helps determine which service end point to call for model training -//format is of type "packageName/serviceName/MethodName", Example :"/example_service.Calculator/estimate_add" -//Daemon will invoke the model training end point , when the below method option is specified -message TrainingMethodOption { - string trainingMethodIndicator = 1; -} - -extend google.protobuf.MethodOptions { - TrainingMethodOption my_method_option = 9999197; -} - -message UpdateModelRequest { - ModelDetails update_model_details = 1 ; - AuthorizationDetails authorization = 2; -} - - -message ModelDetailsResponse { - Status status = 1; - ModelDetails model_details = 2; -} - -service Model { - - // The AI developer needs to Implement this service (do not copy this in your service proto) and Daemon will call these - // There will be no cost borne by the consumer in calling these methods, - // Pricing will apply when you actually call the training methods defined. - // AI consumer will call all these methods - rpc create_model(CreateModelRequest) returns (ModelDetailsResponse) {} - rpc delete_model(UpdateModelRequest) returns (ModelDetailsResponse) {} - rpc get_model_status(ModelDetailsRequest) returns (ModelDetailsResponse) {} - - - // Daemon will implement, however the AI developer should skip implementing these and just provide dummy code. - rpc update_model_access(UpdateModelRequest) returns (ModelDetailsResponse) {} - rpc get_all_models(AccessibleModelsRequest) returns (AccessibleModelsResponse) {} - - -} diff --git a/training/training_daemon.proto b/training/training_daemon.proto new file mode 100644 index 00000000..b130baa3 --- /dev/null +++ b/training/training_daemon.proto @@ -0,0 +1,124 @@ +syntax = "proto3"; +package training_daemon; +import "google/protobuf/descriptor.proto"; // Required for indicators to work +import "google/protobuf/struct.proto"; // Required for google.protobuf.ListValue +import "training_v2.proto"; +option go_package = "../training"; +import "google/protobuf/empty.proto"; + +message AuthorizationDetails { + uint64 current_block = 1; // Check for relevance within a range of +/- N blocks + // Signer can specify any message here + string message = 2; + // Signature of the following message: + // ("user specified message", user_address, current_block_number) + bytes signature = 3; + string signer_address = 4; +} + +message NewModelRequest { + AuthorizationDetails authorization = 1; + trainingV2.NewModel model = 2; +} + +message AuthValidateRequest { + AuthorizationDetails authorization = 1; + string model_id = 2; + string training_data_link = 3; +} + +message UploadAndValidateRequest { + AuthorizationDetails authorization = 1; + string model_id = 2; + bytes data = 3; +} + +message CommonRequest { + AuthorizationDetails authorization = 1; + string model_id = 2; +} + +message UpdateModelRequest { + AuthorizationDetails authorization = 1; + string model_id = 2; + string model_name = 3; + string description = 4; + repeated string address_list = 5; +} + +message ModelsResponse { + repeated trainingV2.ModelResponse list_of_models = 1; +} + +// These methods are implemented only by the daemon; the service provider should ignore them +service Daemon { + // Free + rpc create_model(NewModelRequest) returns (trainingV2.ModelResponse) {} + + // Free + rpc validate_model_price(AuthValidateRequest) returns (trainingV2.PriceInBaseUnit) {} + + // Paid + rpc upload_and_validate(stream UploadAndValidateRequest) returns (trainingV2.StatusResponse) {} + + // Paid + rpc validate_model(AuthValidateRequest) returns (trainingV2.StatusResponse) {} + + // Free, one signature for both train_model_price & train_model methods + rpc train_model_price(CommonRequest) returns (trainingV2.PriceInBaseUnit) {} + + // Paid + rpc train_model(CommonRequest) returns (trainingV2.StatusResponse) {} + + // Free + // After deleting the model, the status becomes DELETED in etcd + rpc delete_model(CommonRequest) returns (trainingV2.StatusResponse) {} + + rpc get_all_models(AllModelsRequest) returns (ModelsResponse) {} + + rpc get_model(CommonRequest) returns (trainingV2.ModelResponse) {} + + rpc update_model(UpdateModelRequest) returns (trainingV2.ModelResponse) {} + + // Unique methods by daemon + // One signature for all getters + rpc get_training_metadata(google.protobuf.Empty) returns (TrainingMetadata) {} + + // Free & without authorization + rpc get_method_metadata(MethodMetadataRequest) returns (MethodMetadata) {} +} + +message MethodMetadataRequest { + string model_id = 1; + // Model ID or gRPC method name + string grpc_method_name = 2; + string grpc_service_name = 3; +} + +message AllModelsRequest { + AuthorizationDetails authorization = 1; + // Filters + trainingV2.Status status = 2; + bool is_public = 3; + string name = 4; // Search by name + uint64 page_size = 5; // Pagination + uint64 page = 6; +} + +message TrainingMetadata { + bool trainingEnabled = 1; + bool trainingInProto = 2; + // Key: grpc_service_name, Value: array of grpc_method_name + map trainingMethods = 3; +} + +message MethodMetadata { + string default_model_id = 50001; + uint64 max_models_per_user = 50002; // max models per method & user + uint64 dataset_max_size_mb = 50003; // max size of dataset + uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset + uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset + string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3 + string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar + string dataset_description = 50008; // additional free-form requirements +} diff --git a/training/training_v2.proto b/training/training_v2.proto new file mode 100644 index 00000000..f6e778fa --- /dev/null +++ b/training/training_v2.proto @@ -0,0 +1,119 @@ +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; // Required for indicators to work +package trainingV2; +option go_package = "../training"; + +// Methods that the service provider must implement +service Model { + + // Free + // Can pass the address of the model creator + rpc create_model(NewModel) returns (ModelID) {} + + // Free + rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {} + + // Paid + rpc upload_and_validate(stream DataToUpload) returns (StatusResponse) {} + + // Paid + rpc validate_model(ValidateRequest) returns (StatusResponse) {} + + // Free, one signature for both train_model_price & train_model methods + rpc train_model_price(ModelID) returns (PriceInBaseUnit) {} + + // Paid + rpc train_model(ModelID) returns (StatusResponse) {} + + // Free + rpc delete_model(ModelID) returns (StatusResponse) { + // After model deletion, the status becomes DELETED in etcd + } + + // Free + rpc get_model_status(ModelID) returns (StatusResponse) {} +} + +message ModelResponse { + string model_id = 1; + Status status = 2; + string updated_date = 3; + string name = 4; + string description = 5; + string grpc_method_name = 6; + string grpc_service_name = 7; + + // List of all addresses that will have access to this model + repeated string address_list = 8; + + // Access to the model is granted only for use and viewing + bool is_public = 9; + + string training_data_link = 10; +} + +// Used as input for new_model requests +// The service provider decides whether to use these fields; returning model_id is mandatory +message NewModel { + string name = 1; + string description = 2; + string grpc_method_name = 3; + string grpc_service_name = 4; + + // List of all addresses that will have access to this model + repeated string address_list = 5; + + // Set this to true if you want your model to be accessible by other AI consumers + bool is_public = 6; + + // These parameters will be passed by the daemon + string organization_id = 7; + string service_id = 8; + string group_id = 9; +} + +// This structure must be used by the service provider +message ModelID { + string model_id = 1; +} + +// This structure must be used by the service provider +// Used in the train_model_price method to get the training/validation price +message PriceInBaseUnit { + uint64 price = 1; // cogs, weis, afet, aasi, etc. +} + +enum Status { + CREATED = 0; + VALIDATING = 1; + VALIDATED = 2; + TRAINING = 3; + READY_TO_USE = 4; // After training is completed + ERRORED = 5; + DELETED = 6; +} + +message StatusResponse { + Status status = 1; +} + +message DataToUpload { + string model_id = 2; + bytes data = 3; +} + +message ValidateRequest { + string model_id = 2; + string training_data_link = 3; +} + +extend google.protobuf.MethodOptions { + string default_model_id = 50001; + uint64 max_models_per_user = 50002; // max models per method & user + uint64 dataset_max_size_mb = 50003; // max size of dataset + uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset + uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset + string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3 + string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar + string dataset_description = 50008; // additional free-form requirements +} diff --git a/training/util.go b/training/util.go new file mode 100644 index 00000000..2a9a3574 --- /dev/null +++ b/training/util.go @@ -0,0 +1,68 @@ +package training + +import ( + "bytes" + "github.com/ethereum/go-ethereum/common/math" + "github.com/singnet/snet-daemon/v5/authutils" + "github.com/singnet/snet-daemon/v5/utils" + "math/big" +) + +// message used to sign is of the form ("__create_model", mpe_address, current_block_number) +func (ds *DaemonService) verifySignature(request *AuthorizationDetails) error { + return authutils.VerifySigner(ds.getMessageBytes(request.Message, request), + request.GetSignature(), utils.ToChecksumAddress(request.SignerAddress)) +} + +// "user passed message ", user_address, current_block_number +func (ds *DaemonService) getMessageBytes(prefixMessage string, request *AuthorizationDetails) []byte { + userAddress := utils.ToChecksumAddress(request.SignerAddress) + message := bytes.Join([][]byte{ + []byte(prefixMessage), + userAddress.Bytes(), + math.U256Bytes(big.NewInt(int64(request.CurrentBlock))), + }, nil) + return message +} + +func remove(s []string, r string) []string { + for i, v := range s { + if v == r { + return append(s[:i], s[i+1:]...) + } + } + return s +} + +func difference(oldAddresses []string, newAddresses []string) []string { + var diff []string + for i := 0; i < 2; i++ { + for _, s1 := range oldAddresses { + found := false + for _, s2 := range newAddresses { + if s1 == s2 { + found = true + break + } + } + // String not found. We add it to return slice + if !found { + diff = append(diff, s1) + } + } + // Swap the slices, only if it was the first loop + if i == 0 { + oldAddresses, newAddresses = newAddresses, oldAddresses + } + } + return diff +} + +func isValuePresent(value string, list []string) bool { + for _, v := range list { + if v == value { + return true + } + } + return false +} diff --git a/utils/common.go b/utils/common.go index ee325751..6e6718fa 100644 --- a/utils/common.go +++ b/utils/common.go @@ -6,8 +6,6 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" - "github.com/singnet/snet-daemon/v5/authutils" - "go.uber.org/zap" ) func Serialize(value any) (slice string, err error) { @@ -28,18 +26,6 @@ func Deserialize(slice string, value any) (err error) { return d.Decode(value) } -func VerifySigner(message []byte, signature []byte, signer common.Address) error { - derivedSigner, err := authutils.GetSignerAddressFromMessage(message, signature) - if err != nil { - zap.L().Error(err.Error()) - return err - } - if err = authutils.VerifyAddress(*derivedSigner, signer); err != nil { - return err - } - return nil -} - func ToChecksumAddress(hexAddress string) common.Address { address := common.HexToAddress(hexAddress) mixedAddress := common.NewMixedcaseAddress(address) From 3c446bda0893a00e6659ec6ec3129cfb0abf8681 Mon Sep 17 00:00:00 2001 From: elmiringos Date: Thu, 19 Dec 2024 15:15:19 +0300 Subject: [PATCH 02/22] feat(training): add worker pool for updating validating, training statuses in etcd --- go.mod | 4 -- go.sum | 51 +--------------- training/service.go | 129 +++++++++++++++++++++++++++++++++++---- training/storage.go | 24 +++++++- training/storage_test.go | 9 +-- 5 files changed, 146 insertions(+), 71 deletions(-) diff --git a/go.mod b/go.mod index 79212fdc..ea9bc45e 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.19.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/caddyserver/certmagic v0.21.4 // indirect github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -80,7 +79,6 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -217,14 +215,12 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect - github.com/status-im/keycard-go v0.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/supranational/blst v0.3.13 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect - github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/urfave/cli/v2 v2.25.7 // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect diff --git a/go.sum b/go.sum index 2d199857..ea046a73 100644 --- a/go.sum +++ b/go.sum @@ -75,17 +75,11 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA= -github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.19.1 h1:mv2yVhy96D2CuskLPXnc58oJNMs5PCWjAZuyYU0p12M= github.com/bits-and-blooms/bitset v1.19.1/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= -github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -129,8 +123,6 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A= -github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/bavard v0.1.24 h1:Lfe+bjYbpaoT7K5JTFoMi5wo9V4REGLvQQbHmatoN2I= github.com/consensys/bavard v0.1.24/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= @@ -189,8 +181,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY= -github.com/emicklei/proto v1.13.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/emicklei/proto v1.13.3 h1:h2u3JgIKSBWHI0jTFFAxiIzOQAe+bRGlQ/WlaIa49Uk= github.com/emicklei/proto v1.13.3/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= @@ -200,12 +190,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy4WDs= github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.14.11 h1:8nFDCUUE67rPc6AKxFj7JKaOa2W/W1Rse3oS6LvvxEY= -github.com/ethereum/go-ethereum v1.14.11/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E= github.com/ethereum/go-ethereum v1.14.12 h1:8hl57x77HSUo+cXExrURjU/w1VhL+ShCTJrTwcCQSe4= github.com/ethereum/go-ethereum v1.14.12/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY= -github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= -github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= @@ -229,8 +215,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= -github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= -github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -300,8 +284,6 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4= github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -445,8 +427,6 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6w github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= -github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -540,8 +520,6 @@ github.com/ipfs/go-unixfsnode v1.9.2 h1:0A12BYs4XOtDPJTMlwmNPlllDfqcc4yie4e919hc github.com/ipfs/go-unixfsnode v1.9.2/go.mod h1:v1nuMFHf4QTIhFUdPMvg1nQu7AqDLvIdwyvJ531Ot1U= github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= -github.com/ipfs/kubo v0.32.0 h1:0in5oT7WaCkkJ6JyOFH/NiNFYK7z1oELpFLgKTXU7Ak= -github.com/ipfs/kubo v0.32.0/go.mod h1:vtxKR1IburF+anxjesDMp39LT8H5YYmZV/Yk5t5xNAk= github.com/ipfs/kubo v0.32.1 h1:nkx5qrkMeJ2f1ET7v3vx7U1ycurM0dC9R7AnsuSrNjk= github.com/ipfs/kubo v0.32.1/go.mod h1:7fi1IMPgW5fupyMFUjJ4d4zbvkTEwq6tV3T+EQvtF28= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= @@ -627,8 +605,6 @@ github.com/libp2p/go-libp2p v0.37.0 h1:8K3mcZgwTldydMCNOiNi/ZJrOB9BY+GlI3UxYzxBi github.com/libp2p/go-libp2p v0.37.0/go.mod h1:GOKmSN99scDuYGTwaTbQPR8Nt6dxrK3ue7OjW2NGDg4= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= -github.com/libp2p/go-libp2p-kad-dht v0.28.0 h1:sDqfW784w7CZQLlnMUwfeqWfXcpedKeZIM/B9/w0Tbk= -github.com/libp2p/go-libp2p-kad-dht v0.28.0/go.mod h1:0wHURlSFdAC42+wF7GEmpLoARw8JuS8do2guCtc/Y/w= github.com/libp2p/go-libp2p-kad-dht v0.28.1 h1:DVTfzG8Ybn88g9RycIq47evWCRss5f0Wm8iWtpwyHso= github.com/libp2p/go-libp2p-kad-dht v0.28.1/go.mod h1:0wHURlSFdAC42+wF7GEmpLoARw8JuS8do2guCtc/Y/w= github.com/libp2p/go-libp2p-kbucket v0.6.4 h1:OjfiYxU42TKQSB8t8WYd8MKhYhMJeO2If+NiuKfb6iQ= @@ -661,8 +637,6 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1005,8 +979,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= -github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -1026,7 +998,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -1044,8 +1015,6 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= -github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= @@ -1141,6 +1110,8 @@ go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozR go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= @@ -1198,10 +1169,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1288,8 +1255,6 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1310,8 +1275,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1377,8 +1340,6 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1399,8 +1360,6 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1501,8 +1460,6 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= -google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= -google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= @@ -1526,10 +1483,6 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= -google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/grpc v1.69.0 h1:quSiOM1GJPmPH5XtU+BCoVXcDVJJAzNcoyfC2cCjGkI= google.golang.org/grpc v1.69.0/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/training/service.go b/training/service.go index 5bf1088d..b806a793 100644 --- a/training/service.go +++ b/training/service.go @@ -6,6 +6,13 @@ package training import ( "context" "fmt" + "maps" + "net/url" + "slices" + "strings" + "sync" + "time" + "github.com/bufbuild/protocompile" "github.com/bufbuild/protocompile/linker" "github.com/singnet/snet-daemon/v5/blockchain" @@ -15,13 +22,9 @@ import ( "google.golang.org/protobuf/types/descriptorpb" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/structpb" - "maps" - "net/url" - "slices" - "strings" - "time" _ "embed" + "github.com/singnet/snet-daemon/v5/config" "github.com/singnet/snet-daemon/v5/escrow" "go.uber.org/zap" @@ -60,6 +63,28 @@ type DaemonService struct { methodsMetadata map[string]*MethodMetadata } +func NewDaemonsService( + serviceMetaData *blockchain.ServiceMetadata, + organizationMetaData *blockchain.OrganizationMetaData, + channelService escrow.PaymentChannelService, + storage *ModelStorage, + userStorage *ModelUserStorage, + serviceUrl string, + trainingMedadata *TrainingMetadata, + methodsMetadata map[string]*MethodMetadata, +) *DaemonService { + return &DaemonService{ + serviceMetaData: serviceMetaData, + organizationMetaData: organizationMetaData, + channelService: channelService, + storage: storage, + userStorage: userStorage, + serviceUrl: serviceUrl, + trainingMetadata: trainingMedadata, + methodsMetadata: methodsMetadata, + } +} + func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest) (*ModelResponse, error) { if request == nil || request.Authorization == nil { @@ -121,6 +146,82 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest return modelResponse, err } +func (ds *DaemonService) getTrainingAndValidatingModelIds(orgId, serviceId, groupId string) []string { + // TODO: make real implmentation + return []string{"1", "2", "3"} +} + +func (ds *DaemonService) startUpdateModelStatusWorker(ctx context.Context, modelId string) { + + modelKey := ds.buildModelKey(modelId) + + currentModelData, ok, err := ds.storage.Get(modelKey) + if err != nil { + zap.L().Error("err in getting modelData from storage", zap.Error(err)) + return + } + if !ok { + zap.L().Error("there is no model with such modelKey", zap.Any("modelKey", modelKey)) + return + } + + _, client, err := ds.getServiceClient() + if err != nil { + zap.L().Error("error in gettting service client", zap.Error(err)) + return + } + + response, err := client.GetModelStatus(ctx, &ModelID{ModelId: modelId}) + if response == nil || err != nil { + zap.L().Error("error in invoking GetModelStatus, service-provider should implement it", zap.Error(err)) + return + } + + newModelData := *currentModelData // Shallow copy of the current model data. + // However, it does not create deep copies of any slices contained within ModelData; modifications to the slices in newModelData will affect currentModelData. + newModelData.Status = response.Status + ds.storage.CompareAndSwap(modelKey, currentModelData, &newModelData) +} + +func (ds *DaemonService) worker(ctx context.Context, tasks <-chan string, wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case modelID := <-tasks: + ds.startUpdateModelStatusWorker(ctx, modelID) + case <-ctx.Done(): + return + } + } +} + +func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, interval time.Duration, orgID, serviceID, groupID string) { + ticker := time.NewTicker(interval) + modelIDs := ds.getTrainingAndValidatingModelIds(orgID, serviceID, groupID) + tasks := make(chan string, len(modelIDs)) + numWorkers := 3 + var wg sync.WaitGroup + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go ds.worker(ctx, tasks, &wg) + } + + defer ticker.Stop() + + for { + select { + case <-ticker.C: + for _, modelID := range modelIDs { + tasks <- modelID + } + case <-ctx.Done(): + wg.Wait() + return + } + } +} + func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { panic("implement me") } @@ -183,7 +284,6 @@ func getConnection(endpoint string) (conn *grpc.ClientConn) { } } else { conn, err = grpc.NewClient(passthroughURL.Host, grpc.WithTransportCredentials(insecure.NewCredentials()), options) - if err != nil { zap.L().Panic("error dialing service", zap.Error(err)) } @@ -385,9 +485,8 @@ func (ds DaemonService) updateModelDetailsWithLatestStatus(request *CommonReques if err = ds.storage.Put(key, data); err != nil { zap.L().Error("issue with retrieving model data from storage", zap.Error(err)) } - } else { - zap.L().Error("can't get model data from etcd", zap.Error(err)) } + zap.L().Error("can't get model data from etcd", zap.Error(err)) return } @@ -655,9 +754,8 @@ func NewModelService(channelService escrow.PaymentChannelService, serMetaData *b userStorage: userStorage, serviceUrl: serviceURL, } - } else { - return &NoModelSupportService{} } + return &NoModelSupportService{} } // NewTrainingService daemon self server @@ -674,7 +772,8 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData serviceURL := config.GetString(config.ModelMaintenanceEndPoint) if config.IsValidUrl(serviceURL) && config.GetBool(config.BlockchainEnabledKey) { - return &DaemonService{ + + daemonService := &DaemonService{ channelService: channelService, serviceMetaData: serMetaData, organizationMetaData: orgMetadata, @@ -684,9 +783,13 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData trainingMetadata: &trainMD, methodsMetadata: methodsMD, } - } else { - return &NoTrainingService{} + + daemonService.ManageUpdateModelStatusWorkers(context.Background(), 4*time.Second, "3", "2", "1") + + return daemonService } + + return &NoTrainingService{} } // parseTrainingMetadata TODO add comment diff --git a/training/storage.go b/training/storage.go index dde4428b..095bdbbf 100644 --- a/training/storage.go +++ b/training/storage.go @@ -2,9 +2,10 @@ package training import ( "fmt" + "reflect" + "github.com/singnet/snet-daemon/v5/storage" "github.com/singnet/snet-daemon/v5/utils" - "reflect" ) type ModelStorage struct { @@ -86,6 +87,27 @@ func (data *ModelData) String() string { data.CreatedByAddress, data.UpdatedByAddress, data.Status, data.TrainingLink) } +type TrainingValidatingModelKey struct { + OrganizationId string + ServiceId string + GroupId string +} + +func (key *TrainingValidatingModelKey) String() string { + return fmt.Sprintf("{ID:%v|%v|%v|%v}", + key.OrganizationId, + key.ServiceId, + key.GroupId) +} + +type TrainingValidatingModelsData struct { + ModelIds []string +} + +func (data *TrainingValidatingModelsData) String() string { + return fmt.Sprintf("{DATA:%v}", data.ModelIds) +} + type ModelData struct { IsPublic bool ModelName string diff --git a/training/storage_test.go b/training/storage_test.go index c46862e2..5e3db5fb 100644 --- a/training/storage_test.go +++ b/training/storage_test.go @@ -1,10 +1,11 @@ package training import ( + "testing" + "github.com/singnet/snet-daemon/v5/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "testing" ) type ModelStorageSuite struct { @@ -21,11 +22,11 @@ type ModelStorageSuite struct { func (suite *ModelStorageSuite) getModelKey(modelId string) *ModelKey { return &ModelKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, - ServiceId: suite.serviceId, ModelId: modelId, GRPCMethodName: suite.methodName} + ServiceId: suite.serviceId, ModelId: modelId} } func (suite *ModelStorageSuite) getUserModelKey(address string) *ModelUserKey { return &ModelUserKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, - ServiceId: suite.serviceId, GRPCMethodName: suite.methodName, UserAddress: address} + ServiceId: suite.serviceId, UserAddress: address} } func (suite *ModelStorageSuite) getModelData(modelId string) *ModelData { @@ -48,7 +49,7 @@ func (suite *ModelStorageSuite) getUserModelData(modelId []string) *ModelUserDat OrganizationId: suite.organizationId, ServiceId: suite.serviceId, GroupId: suite.groupId, - GRPCMethodName: suite.methodName, + // GRPCMethodName: suite.methodName, } } func (suite *ModelStorageSuite) SetupSuite() { From 99d71323c63a768647ede5dc1925c53da45ee9c4 Mon Sep 17 00:00:00 2001 From: elmiringos Date: Thu, 19 Dec 2024 15:15:26 +0300 Subject: [PATCH 03/22] test(training): add tests for updating model statuses functionality --- .../integrationtests/test_provider_service.go | 88 ++++++++ .../updating_status_worker_test.go | 210 ++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 training/integrationtests/test_provider_service.go create mode 100644 training/integrationtests/updating_status_worker_test.go diff --git a/training/integrationtests/test_provider_service.go b/training/integrationtests/test_provider_service.go new file mode 100644 index 00000000..59866a3d --- /dev/null +++ b/training/integrationtests/test_provider_service.go @@ -0,0 +1,88 @@ +package integrationtests + +import ( + "context" + "log" + "net" + + "github.com/singnet/snet-daemon/v5/training" + "google.golang.org/grpc" +) + +type model struct { + name string + method string + desc string +} + +func startTestService(address string) { + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + grpcServer := grpc.NewServer() + var trainingServer TrainServer + training.RegisterModelServer(grpcServer, &trainingServer) + + go func() { + log.Println("Starting server on 127.0.0.1:5001") + if err := grpcServer.Serve(lis); err != nil { + log.Fatalf("grpcServer failed: %v", err) + } + }() +} + +type TrainServer struct { + training.UnimplementedModelServer +} + +func (s *TrainServer) CreateModel(ctx context.Context, newModel *training.NewModel) (*training.ModelID, error) { + return &training.ModelID{ + ModelId: "1", // TODO random gen + }, nil +} + +func (s *TrainServer) ValidateModelPrice(ctx context.Context, request *training.ValidateRequest) (*training.PriceInBaseUnit, error) { + return &training.PriceInBaseUnit{ + Price: 1, + }, nil +} + +func (s *TrainServer) UploadAndValidate(server training.Model_UploadAndValidateServer) error { + panic("implement me") +} + +func (s *TrainServer) ValidateModel(ctx context.Context, request *training.ValidateRequest) (*training.StatusResponse, error) { + return &training.StatusResponse{ + Status: training.Status_VALIDATING, + }, nil +} + +func (s *TrainServer) TrainModelPrice(ctx context.Context, id *training.ModelID) (*training.PriceInBaseUnit, error) { + return &training.PriceInBaseUnit{ + Price: 1, + }, nil +} + +func (s *TrainServer) TrainModel(ctx context.Context, id *training.ModelID) (*training.StatusResponse, error) { + return &training.StatusResponse{ + Status: training.Status_TRAINING, + }, nil +} + +func (s *TrainServer) DeleteModel(ctx context.Context, id *training.ModelID) (*training.StatusResponse, error) { + return &training.StatusResponse{ + Status: training.Status_DELETED, + }, nil +} + +func (s *TrainServer) GetModelStatus(ctx context.Context, id *training.ModelID) (*training.StatusResponse, error) { + return &training.StatusResponse{ + Status: training.Status_VALIDATED, + }, nil +} + +func (s *TrainServer) mustEmbedUnimplementedModelServer() { + panic("implement me") +} diff --git a/training/integrationtests/updating_status_worker_test.go b/training/integrationtests/updating_status_worker_test.go new file mode 100644 index 00000000..bfc381be --- /dev/null +++ b/training/integrationtests/updating_status_worker_test.go @@ -0,0 +1,210 @@ +package integrationtests + +import ( + "context" + "testing" + "time" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/singnet/snet-daemon/v5/blockchain" + "github.com/singnet/snet-daemon/v5/config" + "github.com/singnet/snet-daemon/v5/storage" + "github.com/singnet/snet-daemon/v5/training" +) + +func setupTestConfig() { + testConfigJson := ` +{ + "blockchain_enabled": true, + "blockchain_network_selected": "sepolia", + "daemon_end_point": "127.0.0.1:8080", + "daemon_group_name":"default_group", + "payment_channel_storage_type": "etcd", + "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "ipfs_timeout" : 30, + "passthrough_enabled": true, + "passthrough_endpoint":"http://127.0.0.1:5002", + "service_id": "service_id", + "organization_id": "test_org_id", + "metering_enabled": false, + "ssl_cert": "", + "ssl_key": "", + "max_message_size_in_mb" : 4, + "daemon_type": "grpc", + "enable_dynamic_pricing":false, + "allowed_user_flag" :false, + "auto_ssl_domain": "", + "auto_ssl_cache_dir": ".certs", + "private_key": "", + "log": { + "level": "info", + "timezone": "UTC", + "formatter": { + "type": "text", + "timestamp_format": "2006-01-02T15:04:05.999Z07:00" + }, + "output": { + "type": ["file", "stdout"], + "file_pattern": "./snet-daemon.%Y%m%d.log", + "current_link": "./snet-daemon.log", + "max_size_in_mb": 10, + "max_age_in_days": 7, + "rotation_count": 0 + }, + "hooks": [] + }, + "payment_channel_storage_client": { + "connection_timeout": "0s", + "request_timeout": "0s", + "hot_reload": true + }, + "payment_channel_storage_server": { + "id": "storage-1", + "scheme": "http", + "host" : "127.0.0.1", + "client_port": 2379, + "peer_port": 2380, + "token": "unique-token", + "cluster": "storage-1=http://127.0.0.1:2380", + "startup_timeout": "1m", + "data_dir": "storage-data-dir-1.etcd", + "log_level": "info", + "log_outputs": ["./etcd-server.log"], + "enabled": false + }, + "alerts_email": "", + "service_heartbeat_type": "http", + "token_expiry_in_minutes": 1440, + "model_training_enabled": false +}` + + var testConfig = viper.New() + err := config.ReadConfigFromJsonString(testConfig, testConfigJson) + if err != nil { + zap.L().Fatal("Error in reading config") + } + + config.SetVip(testConfig) +} + +var testJsonOrgGroupData = "{ \"org_name\": \"organization_name\", \"org_id\": \"test_org_id\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"license_server_endpoints\": [\"https://licensendpoint:8082\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" + +func addTestModels() (*training.ModelKey, *training.ModelKey, map[string]training.ModelData, *training.ModelStorage) { + memStorage := storage.NewMemStorage() + modelStorage := training.NewModelStorage(memStorage) + + modelA := &training.ModelData{ + IsPublic: true, + ModelName: "testModel", + AuthorizedAddresses: []string{}, + Status: training.Status_CREATED, + CreatedByAddress: "address", + ModelId: "1", + UpdatedByAddress: "string", + GroupId: "string", + OrganizationId: "string", + ServiceId: "string", + GRPCMethodName: "string", + GRPCServiceName: "string", + Description: "string", + IsDefault: true, + TrainingLink: "string", + UpdatedDate: "string", + } + + modelAKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: "1", + } + + modelB := &training.ModelData{ + IsPublic: true, + ModelName: "testModel", + AuthorizedAddresses: []string{}, + Status: training.Status_CREATED, + CreatedByAddress: "address", + ModelId: "1", + UpdatedByAddress: "string", + GroupId: "string", + OrganizationId: "string", + ServiceId: "string", + GRPCMethodName: "string", + GRPCServiceName: "string", + Description: "string", + IsDefault: true, + TrainingLink: "string", + UpdatedDate: "string", + } + + modelBKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: "2", + } + + modelStorage.Put(modelAKey, modelA) + modelStorage.Put(modelBKey, modelB) + + models := map[string]training.ModelData{ + "1": *modelA, + "2": *modelB, + } + + modelIdsData := "{DATA:[1, 2]}" + + key := &training.TrainingValidatingModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + } + + memStorage.Put(key.String(), modelIdsData) + + return modelAKey, modelBKey, models, modelStorage +} + +func TestRunDaemonService(t *testing.T) { + setupTestConfig() + address := "localhost:5001" + + modelAKey, modelBKey, _, modelStorage := addTestModels() + startTestService(address) + + orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) + assert.Nil(t, err) + + ds := training.NewDaemonsService( + nil, orgMetadata, nil, modelStorage, nil, "http://localhost:5001", nil, nil, + ) + assert.NotNil(t, ds) + + duration := time.Second * 10 + deadline := time.Now().Add(duration) + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + ds.ManageUpdateModelStatusWorkers(ctx, time.Second*3, "test_org_id", "service_id", "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=") + + select { + case <-ctx.Done(): + t.Logf("Context done: %v", ctx.Err()) + case <-time.After(duration): + t.Logf("Operation timed out after %v", duration) + } + + model1Data, ok, err := modelStorage.Get(modelAKey) + assert.Nil(t, err) + assert.True(t, ok) + assert.Equal(t, training.Status_VALIDATED, model1Data.Status) + + model2Data, ok, err := modelStorage.Get(modelBKey) + assert.Nil(t, err) + assert.True(t, ok) + assert.Equal(t, training.Status_VALIDATED, model2Data.Status) +} From f0834c13fff0b20b95681fbb12f35619ef637060 Mon Sep 17 00:00:00 2001 From: elmiringos Date: Tue, 24 Dec 2024 17:05:15 +0300 Subject: [PATCH 04/22] feat(training): add new storage type - pending model storage for models that have training\validatin status --- snetd/cmd/components.go | 2 +- training/service.go | 53 +++++++++++--- training/storage.go | 155 ++++++++++++++++++++++++++-------------- 3 files changed, 148 insertions(+), 62 deletions(-) diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index 10acf943..fa2b10f5 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -564,7 +564,7 @@ func (components *Components) ModelUserStorage() *training.ModelUserStorage { if components.modelUserStorage != nil { return components.modelUserStorage } - components.modelUserStorage = training.NewUerModelStorage(components.AtomicStorage()) + components.modelUserStorage = training.NewUserModelStorage(components.AtomicStorage()) return components.modelUserStorage } diff --git a/training/service.go b/training/service.go index b806a793..6f0053ec 100644 --- a/training/service.go +++ b/training/service.go @@ -58,6 +58,7 @@ type DaemonService struct { channelService escrow.PaymentChannelService storage *ModelStorage userStorage *ModelUserStorage + pendingStorage *PendingModelStorage serviceUrl string trainingMetadata *TrainingMetadata methodsMetadata map[string]*MethodMetadata @@ -69,6 +70,7 @@ func NewDaemonsService( channelService escrow.PaymentChannelService, storage *ModelStorage, userStorage *ModelUserStorage, + pendingStorage *PendingModelStorage, serviceUrl string, trainingMedadata *TrainingMetadata, methodsMetadata map[string]*MethodMetadata, @@ -79,6 +81,7 @@ func NewDaemonsService( channelService: channelService, storage: storage, userStorage: userStorage, + pendingStorage: pendingStorage, serviceUrl: serviceUrl, trainingMetadata: trainingMedadata, methodsMetadata: methodsMetadata, @@ -146,9 +149,33 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest return modelResponse, err } -func (ds *DaemonService) getTrainingAndValidatingModelIds(orgId, serviceId, groupId string) []string { - // TODO: make real implmentation - return []string{"1", "2", "3"} +func (ds *DaemonService) buildPendingModelKey() *PendingModelKey { + return &PendingModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + } +} + +func (ds *DaemonService) setPendingModelIds(modelIds *[]string) { + key := ds.buildPendingModelKey() + + data := &PendingModelData{ + ModelIDs: *modelIds, + } + + ds.pendingStorage.Put(key, data) +} + +func (ds *DaemonService) getPendingModelIds() (*PendingModelData, error) { + key := ds.buildPendingModelKey() + + data, _, err := ds.pendingStorage.Get(key) + if err != nil { + return nil, err + } + + return data, nil } func (ds *DaemonService) startUpdateModelStatusWorker(ctx context.Context, modelId string) { @@ -183,7 +210,7 @@ func (ds *DaemonService) startUpdateModelStatusWorker(ctx context.Context, model ds.storage.CompareAndSwap(modelKey, currentModelData, &newModelData) } -func (ds *DaemonService) worker(ctx context.Context, tasks <-chan string, wg *sync.WaitGroup) { +func (ds *DaemonService) updateModelStatusworker(ctx context.Context, tasks <-chan string, wg *sync.WaitGroup) { defer wg.Done() for { select { @@ -197,14 +224,22 @@ func (ds *DaemonService) worker(ctx context.Context, tasks <-chan string, wg *sy func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, interval time.Duration, orgID, serviceID, groupID string) { ticker := time.NewTicker(interval) - modelIDs := ds.getTrainingAndValidatingModelIds(orgID, serviceID, groupID) - tasks := make(chan string, len(modelIDs)) + data, err := ds.getPendingModelIds() + if err != nil { + zap.L().Error("Error in getting pending model IDs", zap.Error(err)) + return + } + if data == nil { + zap.L().Debug("There are no pending models") + return + } + tasks := make(chan string, len(data.ModelIDs)) numWorkers := 3 var wg sync.WaitGroup for i := 0; i < numWorkers; i++ { wg.Add(1) - go ds.worker(ctx, tasks, &wg) + go ds.updateModelStatusworker(ctx, tasks, &wg) } defer ticker.Stop() @@ -212,7 +247,7 @@ func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, int for { select { case <-ticker.C: - for _, modelID := range modelIDs { + for _, modelID := range data.ModelIDs { tasks <- modelID } case <-ctx.Done(): @@ -784,7 +819,7 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData methodsMetadata: methodsMD, } - daemonService.ManageUpdateModelStatusWorkers(context.Background(), 4*time.Second, "3", "2", "1") + go daemonService.ManageUpdateModelStatusWorkers(context.Background(), 4*time.Second, "3", "2", "1") return daemonService } diff --git a/training/storage.go b/training/storage.go index 095bdbbf..214fcea8 100644 --- a/training/storage.go +++ b/training/storage.go @@ -16,7 +16,11 @@ type ModelUserStorage struct { delegate storage.TypedAtomicStorage } -func NewUerModelStorage(atomicStorage storage.AtomicStorage) *ModelUserStorage { +type PendingModelStorage struct { + delegate storage.TypedAtomicStorage +} + +func NewUserModelStorage(atomicStorage storage.AtomicStorage) *ModelUserStorage { prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/userModelStorage") userModelStorage := storage.NewTypedAtomicStorageImpl( prefixedStorage, serializeModelUserKey, reflect.TypeOf(ModelUserKey{}), utils.Serialize, utils.Deserialize, @@ -34,6 +38,55 @@ func NewModelStorage(atomicStorage storage.AtomicStorage) *ModelStorage { return &ModelStorage{delegate: modelStorage} } +func NewPendingModelStorage(atomicStorage storage.AtomicStorage) *PendingModelStorage { + prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/pendingModelStorage") + pendingModelStorage := storage.NewTypedAtomicStorageImpl( + prefixedStorage, serializePendingModelKey, reflect.TypeOf(PendingModelKey{}), utils.Serialize, utils.Deserialize, + reflect.TypeOf(PendingModelData{}), + ) + return &PendingModelStorage{delegate: pendingModelStorage} +} + +type ModelKey struct { + OrganizationId string + ServiceId string + GroupId string + //GRPCMethodName string + //GRPCServiceName string + ModelId string +} + +func (key *ModelKey) String() string { + return fmt.Sprintf("{ID:%v|%v|%v|%v}", key.OrganizationId, + key.ServiceId, key.GroupId, key.ModelId) +} + +type ModelData struct { + IsPublic bool + ModelName string + AuthorizedAddresses []string + Status Status + CreatedByAddress string + ModelId string + UpdatedByAddress string + GroupId string + OrganizationId string + ServiceId string + GRPCMethodName string + GRPCServiceName string + Description string + IsDefault bool + TrainingLink string + UpdatedDate string +} + +func (data *ModelData) String() string { + return fmt.Sprintf("{DATA:%v|%v|%v|%v|%v|%v|IsPublic:%v|accesibleAddress:%v|createdBy:%v|updatedBy:%v|status:%v|TrainingLin:%v}", + data.OrganizationId, + data.ServiceId, data.GroupId, data.GRPCServiceName, data.GRPCMethodName, data.ModelId, data.AuthorizedAddresses, data.IsPublic, + data.CreatedByAddress, data.UpdatedByAddress, data.Status, data.TrainingLink) +} + type ModelUserKey struct { OrganizationId string ServiceId string @@ -66,70 +119,31 @@ func (data *ModelUserData) String() string { data.ServiceId, data.GroupId, data.UserAddress, data.ModelIds) } -type ModelKey struct { +type PendingModelKey struct { OrganizationId string ServiceId string GroupId string - //GRPCMethodName string - //GRPCServiceName string - ModelId string } -func (key *ModelKey) String() string { - return fmt.Sprintf("{ID:%v|%v|%v|%v}", key.OrganizationId, - key.ServiceId, key.GroupId, key.ModelId) -} - -func (data *ModelData) String() string { - return fmt.Sprintf("{DATA:%v|%v|%v|%v|%v|%v|IsPublic:%v|accesibleAddress:%v|createdBy:%v|updatedBy:%v|status:%v|TrainingLin:%v}", - data.OrganizationId, - data.ServiceId, data.GroupId, data.GRPCServiceName, data.GRPCMethodName, data.ModelId, data.AuthorizedAddresses, data.IsPublic, - data.CreatedByAddress, data.UpdatedByAddress, data.Status, data.TrainingLink) -} - -type TrainingValidatingModelKey struct { - OrganizationId string - ServiceId string - GroupId string -} - -func (key *TrainingValidatingModelKey) String() string { - return fmt.Sprintf("{ID:%v|%v|%v|%v}", +func (key *PendingModelKey) String() string { + return fmt.Sprintf("{ID:%v|%v|%v}", key.OrganizationId, key.ServiceId, key.GroupId) } -type TrainingValidatingModelsData struct { - ModelIds []string +type PendingModelData struct { + ModelIDs []string } -func (data *TrainingValidatingModelsData) String() string { - return fmt.Sprintf("{DATA:%v}", data.ModelIds) -} - -type ModelData struct { - IsPublic bool - ModelName string - AuthorizedAddresses []string - Status Status - CreatedByAddress string - ModelId string - UpdatedByAddress string - GroupId string - OrganizationId string - ServiceId string - GRPCMethodName string - GRPCServiceName string - Description string - IsDefault bool - TrainingLink string - UpdatedDate string +// PendingModelData maintain the list of all modelIds that have TRAINING\VALIDATING status +func (data *PendingModelData) String() string { + return fmt.Sprintf("{DATA:%v}", data.ModelIDs) } func serializeModelKey(key any) (serialized string, err error) { - myKey := key.(*ModelKey) - return myKey.String(), nil + modelKey := key.(*ModelKey) + return modelKey.String(), nil } func (storage *ModelStorage) Get(key *ModelKey) (state *ModelData, ok bool, err error) { @@ -161,9 +175,10 @@ func (storage *ModelStorage) CompareAndSwap(key *ModelKey, prevState *ModelData, newState *ModelData) (ok bool, err error) { return storage.delegate.CompareAndSwap(key, prevState, newState) } + func serializeModelUserKey(key any) (serialized string, err error) { - myKey := key.(*ModelUserKey) - return myKey.String(), nil + modelUserKey := key.(*ModelUserKey) + return modelUserKey.String(), nil } func (storage *ModelUserStorage) Get(key *ModelUserKey) (state *ModelUserData, ok bool, err error) { @@ -195,3 +210,39 @@ func (storage *ModelUserStorage) CompareAndSwap(key *ModelUserKey, prevState *Mo newState *ModelUserData) (ok bool, err error) { return storage.delegate.CompareAndSwap(key, prevState, newState) } + +func serializePendingModelKey(key any) (serialized string, err error) { + pendingModelKey := key.(*PendingModelKey) + return pendingModelKey.String(), nil +} + +func (storage *PendingModelStorage) Get(key *PendingModelKey) (state *PendingModelData, ok bool, err error) { + value, ok, err := storage.delegate.Get(key) + if err != nil || !ok { + return nil, ok, err + } + + return value.(*PendingModelData), ok, err +} + +func (storage *PendingModelStorage) GetAll() (states []*PendingModelData, err error) { + values, err := storage.delegate.GetAll() + if err != nil { + return + } + + return values.([]*PendingModelData), nil +} + +func (storage *PendingModelStorage) Put(key *PendingModelKey, state *PendingModelData) (err error) { + return storage.delegate.Put(key, state) +} + +func (storage *PendingModelStorage) PutIfAbsent(key *PendingModelKey, state *PendingModelData) (ok bool, err error) { + return storage.delegate.PutIfAbsent(key, state) +} + +func (storage *PendingModelStorage) CompareAndSwap(key *PendingModelKey, prevState *PendingModelData, + newState *PendingModelData) (ok bool, err error) { + return storage.delegate.CompareAndSwap(key, prevState, newState) +} From 2376dbc9e4082ac9de02a94eb558d9635157cca1 Mon Sep 17 00:00:00 2001 From: elmiringos Date: Tue, 24 Dec 2024 17:07:01 +0300 Subject: [PATCH 05/22] test(training): update integration tests for ManageUpdateModelStatusWorkers, add unit test for pending model storage --- .../integration}/test_provider_service.go | 2 +- .../updating_status_worker_test.go | 21 +++-- training/{ => tests/unit}/storage_test.go | 85 +++++++++++++------ 3 files changed, 71 insertions(+), 37 deletions(-) rename training/{integrationtests => tests/integration}/test_provider_service.go (98%) rename training/{integrationtests => tests/integration}/updating_status_worker_test.go (91%) rename training/{ => tests/unit}/storage_test.go (62%) diff --git a/training/integrationtests/test_provider_service.go b/training/tests/integration/test_provider_service.go similarity index 98% rename from training/integrationtests/test_provider_service.go rename to training/tests/integration/test_provider_service.go index 59866a3d..c5a8e86e 100644 --- a/training/integrationtests/test_provider_service.go +++ b/training/tests/integration/test_provider_service.go @@ -1,4 +1,4 @@ -package integrationtests +package tests import ( "context" diff --git a/training/integrationtests/updating_status_worker_test.go b/training/tests/integration/updating_status_worker_test.go similarity index 91% rename from training/integrationtests/updating_status_worker_test.go rename to training/tests/integration/updating_status_worker_test.go index bfc381be..f049adcf 100644 --- a/training/integrationtests/updating_status_worker_test.go +++ b/training/tests/integration/updating_status_worker_test.go @@ -1,4 +1,4 @@ -package integrationtests +package tests import ( "context" @@ -92,9 +92,10 @@ func setupTestConfig() { var testJsonOrgGroupData = "{ \"org_name\": \"organization_name\", \"org_id\": \"test_org_id\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"license_server_endpoints\": [\"https://licensendpoint:8082\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" -func addTestModels() (*training.ModelKey, *training.ModelKey, map[string]training.ModelData, *training.ModelStorage) { +func addTestModels() (*training.ModelKey, *training.ModelKey, map[string]training.ModelData, *training.ModelStorage, *training.PendingModelStorage) { memStorage := storage.NewMemStorage() modelStorage := training.NewModelStorage(memStorage) + pendingModelStorage := training.NewPendingModelStorage(memStorage) modelA := &training.ModelData{ IsPublic: true, @@ -156,31 +157,33 @@ func addTestModels() (*training.ModelKey, *training.ModelKey, map[string]trainin "2": *modelB, } - modelIdsData := "{DATA:[1, 2]}" - - key := &training.TrainingValidatingModelKey{ + key := &training.PendingModelKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", } - memStorage.Put(key.String(), modelIdsData) + pendingModelsData := &training.PendingModelData{ + ModelIDs: []string{"1", "2"}, + } + + pendingModelStorage.Put(key, pendingModelsData) - return modelAKey, modelBKey, models, modelStorage + return modelAKey, modelBKey, models, modelStorage, pendingModelStorage } func TestRunDaemonService(t *testing.T) { setupTestConfig() address := "localhost:5001" - modelAKey, modelBKey, _, modelStorage := addTestModels() + modelAKey, modelBKey, _, modelStorage, pendingModelStorage := addTestModels() startTestService(address) orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) assert.Nil(t, err) ds := training.NewDaemonsService( - nil, orgMetadata, nil, modelStorage, nil, "http://localhost:5001", nil, nil, + nil, orgMetadata, nil, modelStorage, nil, pendingModelStorage, "http://localhost:5001", nil, nil, ) assert.NotNil(t, ds) diff --git a/training/storage_test.go b/training/tests/unit/storage_test.go similarity index 62% rename from training/storage_test.go rename to training/tests/unit/storage_test.go index 5e3db5fb..e582122e 100644 --- a/training/storage_test.go +++ b/training/tests/unit/storage_test.go @@ -1,18 +1,20 @@ -package training +package tests import ( "testing" - "github.com/singnet/snet-daemon/v5/storage" + base_storage "github.com/singnet/snet-daemon/v5/storage" + "github.com/singnet/snet-daemon/v5/training" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type ModelStorageSuite struct { suite.Suite - memoryStorage *storage.MemoryStorage - storage *ModelStorage - userstorage *ModelUserStorage + memoryStorage *base_storage.MemoryStorage + storage *training.ModelStorage + userStorage *training.ModelUserStorage + pendingStorage *training.PendingModelStorage organizationId string serviceId string groupId string @@ -20,18 +22,24 @@ type ModelStorageSuite struct { accessibleAddress []string } -func (suite *ModelStorageSuite) getModelKey(modelId string) *ModelKey { - return &ModelKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, +func (suite *ModelStorageSuite) getModelKey(modelId string) *training.ModelKey { + return &training.ModelKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, ServiceId: suite.serviceId, ModelId: modelId} } -func (suite *ModelStorageSuite) getUserModelKey(address string) *ModelUserKey { - return &ModelUserKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, + +func (suite *ModelStorageSuite) getUserModelKey(address string) *training.ModelUserKey { + return &training.ModelUserKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, ServiceId: suite.serviceId, UserAddress: address} } -func (suite *ModelStorageSuite) getModelData(modelId string) *ModelData { - return &ModelData{ - Status: Status_CREATED, +func (suite *ModelStorageSuite) getPendingModelKey() *training.PendingModelKey { + return &training.PendingModelKey{OrganizationId: suite.organizationId, ServiceId: suite.serviceId, + GroupId: suite.groupId} +} + +func (suite *ModelStorageSuite) getModelData(modelId string) *training.ModelData { + return &training.ModelData{ + Status: training.Status_CREATED, ModelId: modelId, OrganizationId: suite.organizationId, ServiceId: suite.serviceId, @@ -43,8 +51,8 @@ func (suite *ModelStorageSuite) getModelData(modelId string) *ModelData { } } -func (suite *ModelStorageSuite) getUserModelData(modelId []string) *ModelUserData { - return &ModelUserData{ +func (suite *ModelStorageSuite) getUserModelData(modelId []string) *training.ModelUserData { + return &training.ModelUserData{ ModelIds: modelId, OrganizationId: suite.organizationId, ServiceId: suite.serviceId, @@ -52,13 +60,24 @@ func (suite *ModelStorageSuite) getUserModelData(modelId []string) *ModelUserDat // GRPCMethodName: suite.methodName, } } + +func (suite *ModelStorageSuite) getPendingModelData(modelIds []string) *training.PendingModelData { + return &training.PendingModelData{ + ModelIDs: modelIds, + } +} + func (suite *ModelStorageSuite) SetupSuite() { - suite.memoryStorage = storage.NewMemStorage() - suite.storage = NewModelStorage(suite.memoryStorage) - suite.userstorage = NewUerModelStorage(storage.NewMemStorage()) + suite.memoryStorage = base_storage.NewMemStorage() + suite.storage = training.NewModelStorage(suite.memoryStorage) + suite.userStorage = training.NewUserModelStorage(base_storage.NewMemStorage()) + suite.pendingStorage = training.NewPendingModelStorage(base_storage.NewMemStorage()) suite.accessibleAddress = make([]string, 2) suite.accessibleAddress[0] = "ADD1" suite.accessibleAddress[1] = "ADD2" + suite.organizationId = "org_id" + suite.serviceId = "service_id" + suite.groupId = "group_id" } func TestFreeCallUserStorageSuite(t *testing.T) { @@ -126,9 +145,9 @@ func (suite *ModelStorageSuite) TestModelUserStorage_GetAll() { data1 := suite.getUserModelData([]string{"1"}) data2 := suite.getUserModelData([]string{"2"}) data3 := suite.getUserModelData([]string{"3"}) - suite.userstorage.Put(key1, data1) - suite.userstorage.Put(key2, data2) - models, err := suite.userstorage.GetAll() + suite.userStorage.Put(key1, data1) + suite.userStorage.Put(key2, data2) + models, err := suite.userStorage.GetAll() assert.Equal(suite.T(), len(models), 2) assert.Equal(suite.T(), err, nil) match1 := false @@ -143,25 +162,37 @@ func (suite *ModelStorageSuite) TestModelUserStorage_GetAll() { } assert.True(suite.T(), match2) assert.True(suite.T(), match1) - suite.userstorage.PutIfAbsent(key1, data3) - retrieveddata, ok, err := suite.userstorage.Get(key1) + suite.userStorage.PutIfAbsent(key1, data3) + retrieveddata, ok, err := suite.userStorage.Get(key1) assert.True(suite.T(), ok) assert.Nil(suite.T(), err) assert.Equal(suite.T(), retrieveddata, suite.getUserModelData([]string{"1"})) - suite.userstorage.PutIfAbsent(key3, data3) - retrieveddata, ok, err = suite.userstorage.Get(key3) + suite.userStorage.PutIfAbsent(key3, data3) + retrieveddata, ok, err = suite.userStorage.Get(key3) assert.True(suite.T(), ok) assert.Equal(suite.T(), retrieveddata, suite.getUserModelData([]string{"3"})) - ok, err = suite.userstorage.CompareAndSwap(key1, data1, data3) + ok, err = suite.userStorage.CompareAndSwap(key1, data1, data3) assert.True(suite.T(), ok) assert.Nil(suite.T(), err) - retrieveddata, ok, err = suite.userstorage.Get(key1) + retrieveddata, ok, err = suite.userStorage.Get(key1) assert.True(suite.T(), ok) assert.Equal(suite.T(), retrieveddata, suite.getUserModelData([]string{"3"})) - _, ok, err = suite.userstorage.Get(suite.getUserModelKey("4")) + _, ok, err = suite.userStorage.Get(suite.getUserModelKey("4")) assert.Equal(suite.T(), ok, false) +} + +func (suite *ModelStorageSuite) TestPendingModelStorage_Get() { + key := suite.getPendingModelKey() + data := suite.getPendingModelData([]string{"1", "2"}) + err := suite.pendingStorage.Put(key, data) + assert.NoError(suite.T(), err) + + newData, ok, err := suite.pendingStorage.Get(key) + assert.True(suite.T(), ok) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), data, newData) } From c029a46b7ce34f82c7fcb2e25bc6d0f16b711fd3 Mon Sep 17 00:00:00 2001 From: elmiringos Date: Wed, 25 Dec 2024 13:43:39 +0300 Subject: [PATCH 06/22] test(training): add key serialization test, update test provider service, add daemon service suite --- training/storage.go | 5 +- .../tests/integration/daemon_service_test.go | 237 ++++++++++++++++++ .../integration/test_provider_service.go | 46 +++- .../updating_status_worker_test.go | 213 ---------------- training/tests/unit/storage_test.go | 26 ++ 5 files changed, 303 insertions(+), 224 deletions(-) create mode 100644 training/tests/integration/daemon_service_test.go delete mode 100644 training/tests/integration/updating_status_worker_test.go diff --git a/training/storage.go b/training/storage.go index 214fcea8..ec323c2f 100644 --- a/training/storage.go +++ b/training/storage.go @@ -126,10 +126,7 @@ type PendingModelKey struct { } func (key *PendingModelKey) String() string { - return fmt.Sprintf("{ID:%v|%v|%v}", - key.OrganizationId, - key.ServiceId, - key.GroupId) + return fmt.Sprintf("{ID:%v|%v|%v}", key.OrganizationId, key.ServiceId, key.GroupId) } type PendingModelData struct { diff --git a/training/tests/integration/daemon_service_test.go b/training/tests/integration/daemon_service_test.go new file mode 100644 index 00000000..58d8bfc2 --- /dev/null +++ b/training/tests/integration/daemon_service_test.go @@ -0,0 +1,237 @@ +package tests + +import ( + "context" + "testing" + "time" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "google.golang.org/grpc" + + "github.com/singnet/snet-daemon/v5/blockchain" + "github.com/singnet/snet-daemon/v5/config" + "github.com/singnet/snet-daemon/v5/storage" + "github.com/singnet/snet-daemon/v5/training" +) + +type DaemonServiceSuite struct { + suite.Suite + modelStorage *training.ModelStorage + userModelStorage *training.ModelUserStorage + pendingModelStorage *training.PendingModelStorage + daemonService *training.DaemonService + modelKeys []*training.ModelKey + grpcServer *grpc.Server +} + +var ( + testJsonOrgGroupData = "{ \"org_name\": \"organization_name\", \"org_id\": \"test_org_id\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"license_server_endpoints\": [\"https://licensendpoint:8082\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" + testJsonServiceData = "{ \"version\": 1, \"display_name\": \"Example1\", \"encoding\": \"grpc\", \"service_type\": \"grpc\", \"payment_expiration_threshold\": 40320, \"model_ipfs_hash\": \"Qmdiq8Hu6dYiwp712GtnbBxagyfYyvUY1HYqkH7iN76UCc\", " + + " \"mpe_address\": \"0x7E6366Fbe3bdfCE3C906667911FC5237Cc96BD08\", \"groups\": [ { \"free_calls\": 12, \"free_call_signer_address\": \"0x7DF35C98f41F3Af0df1dc4c7F7D4C19a71Dd059F\", \"endpoints\": [\"http://34.344.33.1:2379\",\"http://34.344.33.1:2389\"], \"group_id\": \"88ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\",\"group_name\": \"default_group\", \"pricing\": [ { \"price_model\": \"fixed_price\", \"price_in_cogs\": 2 }, { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"default\":true, \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] }, { \"endpoints\": [\"http://97.344.33.1:2379\",\"http://67.344.33.1:2389\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"pricing\": [ { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] } ] } " +) + +func (suite *DaemonServiceSuite) SetupSuite() { + suite.setupTestConfig() + + modelKeys, modelStorage, userModelStorage, pendingModelStorage := suite.createTestModels() + suite.modelKeys = modelKeys + suite.modelStorage = modelStorage + suite.userModelStorage = userModelStorage + suite.pendingModelStorage = pendingModelStorage + + serviceMetadata, err := blockchain.InitServiceMetaDataFromJson([]byte(testJsonServiceData)) + orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) + if err != nil { + zap.L().Fatal("Error in initinalize organization metadata from json", zap.Error(err)) + } + + suite.daemonService = training.NewDaemonsService( + serviceMetadata, orgMetadata, nil, modelStorage, userModelStorage, pendingModelStorage, "http://localhost:5001", nil, nil, + ) + + address := "localhost:5001" + suite.grpcServer = startTestService(address) +} + +func TestDaemonServiceSuite(t *testing.T) { + suite.Run(t, new(DaemonServiceSuite)) +} + +func (suite *DaemonServiceSuite) setupTestConfig() { + testConfigJson := ` +{ + "blockchain_enabled": true, + "blockchain_network_selected": "sepolia", + "daemon_end_point": "127.0.0.1:8080", + "daemon_group_name":"default_group", + "payment_channel_storage_type": "etcd", + "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "ipfs_timeout" : 30, + "passthrough_enabled": true, + "passthrough_endpoint":"http://127.0.0.1:5002", + "service_id": "service_id", + "organization_id": "test_org_id", + "metering_enabled": false, + "ssl_cert": "", + "ssl_key": "", + "max_message_size_in_mb" : 4, + "daemon_type": "grpc", + "enable_dynamic_pricing":false, + "allowed_user_flag" :false, + "auto_ssl_domain": "", + "auto_ssl_cache_dir": ".certs", + "private_key": "", + "log": { + "level": "info", + "timezone": "UTC", + "formatter": { + "type": "text", + "timestamp_format": "2006-01-02T15:04:05.999Z07:00" + }, + "output": { + "type": ["file", "stdout"], + "file_pattern": "./snet-daemon.%Y%m%d.log", + "current_link": "./snet-daemon.log", + "max_size_in_mb": 10, + "max_age_in_days": 7, + "rotation_count": 0 + }, + "hooks": [] + }, + "payment_channel_storage_client": { + "connection_timeout": "0s", + "request_timeout": "0s", + "hot_reload": true + }, + "payment_channel_storage_server": { + "id": "storage-1", + "scheme": "http", + "host" : "127.0.0.1", + "client_port": 2379, + "peer_port": 2380, + "token": "unique-token", + "cluster": "storage-1=http://127.0.0.1:2380", + "startup_timeout": "1m", + "data_dir": "storage-data-dir-1.etcd", + "log_level": "info", + "log_outputs": ["./etcd-server.log"], + "enabled": false + }, + "alerts_email": "", + "service_heartbeat_type": "http", + "token_expiry_in_minutes": 1440, + "model_training_enabled": false +}` + + var testConfig = viper.New() + err := config.ReadConfigFromJsonString(testConfig, testConfigJson) + if err != nil { + zap.L().Fatal("Error in reading config") + } + + config.SetVip(testConfig) +} + +func (suite *DaemonServiceSuite) createTestModels() ([]*training.ModelKey, *training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage) { + memStorage := storage.NewMemStorage() + modelStorage := training.NewModelStorage(memStorage) + userModelStorage := training.NewUserModelStorage(memStorage) + pendingModelStorage := training.NewPendingModelStorage(memStorage) + + modelA := &training.ModelData{ + IsPublic: true, + ModelName: "testModel", + AuthorizedAddresses: []string{}, + Status: training.Status_CREATED, + CreatedByAddress: "address", + ModelId: "1", + UpdatedByAddress: "string", + GroupId: "string", + OrganizationId: "string", + ServiceId: "string", + GRPCMethodName: "string", + GRPCServiceName: "string", + Description: "string", + IsDefault: true, + TrainingLink: "string", + UpdatedDate: "string", + } + + modelAKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: "1", + } + + modelB := &training.ModelData{ + IsPublic: true, + ModelName: "testModel", + AuthorizedAddresses: []string{}, + Status: training.Status_CREATED, + CreatedByAddress: "address", + ModelId: "1", + UpdatedByAddress: "string", + GroupId: "string", + OrganizationId: "string", + ServiceId: "string", + GRPCMethodName: "string", + GRPCServiceName: "string", + Description: "string", + IsDefault: true, + TrainingLink: "string", + UpdatedDate: "string", + } + + modelBKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: "2", + } + + modelStorage.Put(modelAKey, modelA) + modelStorage.Put(modelBKey, modelB) + modelKeys := []*training.ModelKey{modelAKey, modelBKey} + + key := &training.PendingModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + } + + pendingModelsData := &training.PendingModelData{ + ModelIDs: []string{"1", "2"}, + } + + pendingModelStorage.Put(key, pendingModelsData) + + return modelKeys, modelStorage, userModelStorage, pendingModelStorage +} + +func (suite *DaemonServiceSuite) TestRunDaemonService() { + + duration := time.Second * 10 + deadline := time.Now().Add(duration) + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + suite.daemonService.ManageUpdateModelStatusWorkers(ctx, time.Second*3, "test_org_id", "service_id", "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=") + + select { + case <-ctx.Done(): + zap.L().Info("Context done", zap.Error(ctx.Err())) + case <-time.After(duration): + zap.L().Info("Operation timed out after", zap.Duration("duration", duration)) + } + + for _, modelKey := range suite.modelKeys { + modelData, ok, err := suite.modelStorage.Get(modelKey) + assert.Nil(suite.T(), err) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), training.Status_VALIDATED, modelData.Status) + } +} diff --git a/training/tests/integration/test_provider_service.go b/training/tests/integration/test_provider_service.go index c5a8e86e..62ecd664 100644 --- a/training/tests/integration/test_provider_service.go +++ b/training/tests/integration/test_provider_service.go @@ -2,20 +2,29 @@ package tests import ( "context" + "fmt" "log" "net" "github.com/singnet/snet-daemon/v5/training" + "go.uber.org/zap" "google.golang.org/grpc" ) type model struct { - name string - method string - desc string + modelId string + name string + desc string + grpcMethodName string + grpcServiceName string + addressList []string + isPublic bool + serviceId string + groupId string + status training.Status } -func startTestService(address string) { +func startTestService(address string) *grpc.Server { lis, err := net.Listen("tcp", address) if err != nil { log.Fatalf("failed to listen: %v", err) @@ -25,21 +34,44 @@ func startTestService(address string) { var trainingServer TrainServer training.RegisterModelServer(grpcServer, &trainingServer) + trainingServer.curModelId = 0 + go func() { - log.Println("Starting server on 127.0.0.1:5001") + zap.L().Info("Starting test service", zap.String("address", address)) if err := grpcServer.Serve(lis); err != nil { - log.Fatalf("grpcServer failed: %v", err) + zap.L().Fatal("Error in starting grpcServer", zap.Error(err)) } }() + + return grpcServer } type TrainServer struct { training.UnimplementedModelServer + curModelId int + models []model } func (s *TrainServer) CreateModel(ctx context.Context, newModel *training.NewModel) (*training.ModelID, error) { + modelIdStr := fmt.Sprintf("%v", s.curModelId) + createdModel := &model{ + modelId: modelIdStr, + name: newModel.Name, + desc: newModel.Description, + grpcMethodName: newModel.GrpcMethodName, + grpcServiceName: newModel.GrpcServiceName, + addressList: newModel.AddressList, + isPublic: newModel.IsPublic, + serviceId: newModel.ServiceId, + groupId: newModel.GroupId, + status: training.Status_CREATED, + } + s.models = append(s.models, *createdModel) + + s.curModelId += 1 + return &training.ModelID{ - ModelId: "1", // TODO random gen + ModelId: fmt.Sprintf("%v", s.curModelId), }, nil } diff --git a/training/tests/integration/updating_status_worker_test.go b/training/tests/integration/updating_status_worker_test.go deleted file mode 100644 index f049adcf..00000000 --- a/training/tests/integration/updating_status_worker_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package tests - -import ( - "context" - "testing" - "time" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - "go.uber.org/zap" - - "github.com/singnet/snet-daemon/v5/blockchain" - "github.com/singnet/snet-daemon/v5/config" - "github.com/singnet/snet-daemon/v5/storage" - "github.com/singnet/snet-daemon/v5/training" -) - -func setupTestConfig() { - testConfigJson := ` -{ - "blockchain_enabled": true, - "blockchain_network_selected": "sepolia", - "daemon_end_point": "127.0.0.1:8080", - "daemon_group_name":"default_group", - "payment_channel_storage_type": "etcd", - "ipfs_end_point": "http://ipfs.singularitynet.io:80", - "ipfs_timeout" : 30, - "passthrough_enabled": true, - "passthrough_endpoint":"http://127.0.0.1:5002", - "service_id": "service_id", - "organization_id": "test_org_id", - "metering_enabled": false, - "ssl_cert": "", - "ssl_key": "", - "max_message_size_in_mb" : 4, - "daemon_type": "grpc", - "enable_dynamic_pricing":false, - "allowed_user_flag" :false, - "auto_ssl_domain": "", - "auto_ssl_cache_dir": ".certs", - "private_key": "", - "log": { - "level": "info", - "timezone": "UTC", - "formatter": { - "type": "text", - "timestamp_format": "2006-01-02T15:04:05.999Z07:00" - }, - "output": { - "type": ["file", "stdout"], - "file_pattern": "./snet-daemon.%Y%m%d.log", - "current_link": "./snet-daemon.log", - "max_size_in_mb": 10, - "max_age_in_days": 7, - "rotation_count": 0 - }, - "hooks": [] - }, - "payment_channel_storage_client": { - "connection_timeout": "0s", - "request_timeout": "0s", - "hot_reload": true - }, - "payment_channel_storage_server": { - "id": "storage-1", - "scheme": "http", - "host" : "127.0.0.1", - "client_port": 2379, - "peer_port": 2380, - "token": "unique-token", - "cluster": "storage-1=http://127.0.0.1:2380", - "startup_timeout": "1m", - "data_dir": "storage-data-dir-1.etcd", - "log_level": "info", - "log_outputs": ["./etcd-server.log"], - "enabled": false - }, - "alerts_email": "", - "service_heartbeat_type": "http", - "token_expiry_in_minutes": 1440, - "model_training_enabled": false -}` - - var testConfig = viper.New() - err := config.ReadConfigFromJsonString(testConfig, testConfigJson) - if err != nil { - zap.L().Fatal("Error in reading config") - } - - config.SetVip(testConfig) -} - -var testJsonOrgGroupData = "{ \"org_name\": \"organization_name\", \"org_id\": \"test_org_id\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"license_server_endpoints\": [\"https://licensendpoint:8082\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" - -func addTestModels() (*training.ModelKey, *training.ModelKey, map[string]training.ModelData, *training.ModelStorage, *training.PendingModelStorage) { - memStorage := storage.NewMemStorage() - modelStorage := training.NewModelStorage(memStorage) - pendingModelStorage := training.NewPendingModelStorage(memStorage) - - modelA := &training.ModelData{ - IsPublic: true, - ModelName: "testModel", - AuthorizedAddresses: []string{}, - Status: training.Status_CREATED, - CreatedByAddress: "address", - ModelId: "1", - UpdatedByAddress: "string", - GroupId: "string", - OrganizationId: "string", - ServiceId: "string", - GRPCMethodName: "string", - GRPCServiceName: "string", - Description: "string", - IsDefault: true, - TrainingLink: "string", - UpdatedDate: "string", - } - - modelAKey := &training.ModelKey{ - OrganizationId: "test_org_id", - ServiceId: "service_id", - GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - ModelId: "1", - } - - modelB := &training.ModelData{ - IsPublic: true, - ModelName: "testModel", - AuthorizedAddresses: []string{}, - Status: training.Status_CREATED, - CreatedByAddress: "address", - ModelId: "1", - UpdatedByAddress: "string", - GroupId: "string", - OrganizationId: "string", - ServiceId: "string", - GRPCMethodName: "string", - GRPCServiceName: "string", - Description: "string", - IsDefault: true, - TrainingLink: "string", - UpdatedDate: "string", - } - - modelBKey := &training.ModelKey{ - OrganizationId: "test_org_id", - ServiceId: "service_id", - GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - ModelId: "2", - } - - modelStorage.Put(modelAKey, modelA) - modelStorage.Put(modelBKey, modelB) - - models := map[string]training.ModelData{ - "1": *modelA, - "2": *modelB, - } - - key := &training.PendingModelKey{ - OrganizationId: "test_org_id", - ServiceId: "service_id", - GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - } - - pendingModelsData := &training.PendingModelData{ - ModelIDs: []string{"1", "2"}, - } - - pendingModelStorage.Put(key, pendingModelsData) - - return modelAKey, modelBKey, models, modelStorage, pendingModelStorage -} - -func TestRunDaemonService(t *testing.T) { - setupTestConfig() - address := "localhost:5001" - - modelAKey, modelBKey, _, modelStorage, pendingModelStorage := addTestModels() - startTestService(address) - - orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) - assert.Nil(t, err) - - ds := training.NewDaemonsService( - nil, orgMetadata, nil, modelStorage, nil, pendingModelStorage, "http://localhost:5001", nil, nil, - ) - assert.NotNil(t, ds) - - duration := time.Second * 10 - deadline := time.Now().Add(duration) - ctx, cancel := context.WithDeadline(context.Background(), deadline) - defer cancel() - - ds.ManageUpdateModelStatusWorkers(ctx, time.Second*3, "test_org_id", "service_id", "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=") - - select { - case <-ctx.Done(): - t.Logf("Context done: %v", ctx.Err()) - case <-time.After(duration): - t.Logf("Operation timed out after %v", duration) - } - - model1Data, ok, err := modelStorage.Get(modelAKey) - assert.Nil(t, err) - assert.True(t, ok) - assert.Equal(t, training.Status_VALIDATED, model1Data.Status) - - model2Data, ok, err := modelStorage.Get(modelBKey) - assert.Nil(t, err) - assert.True(t, ok) - assert.Equal(t, training.Status_VALIDATED, model2Data.Status) -} diff --git a/training/tests/unit/storage_test.go b/training/tests/unit/storage_test.go index e582122e..9171ccc8 100644 --- a/training/tests/unit/storage_test.go +++ b/training/tests/unit/storage_test.go @@ -1,6 +1,7 @@ package tests import ( + "fmt" "testing" base_storage "github.com/singnet/snet-daemon/v5/storage" @@ -121,7 +122,32 @@ func (suite *ModelStorageSuite) TestModelStorage_PutIfAbsent() { } func (suite *ModelStorageSuite) Test_serializeModelKey() { + modelId := "1" + expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId, modelId) + key := suite.getModelKey(modelId) + serializedKey := key.String() + + assert.Equal(suite.T(), expectedSerializedKey, serializedKey) +} + +func (suite *ModelStorageSuite) Test_serializeUserModelKey() { + userAddress := "test_address" + expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId, userAddress) + + key := suite.getUserModelKey(userAddress) + serializedKey := key.String() + + assert.Equal(suite.T(), expectedSerializedKey, serializedKey) +} + +func (suite *ModelStorageSuite) Test_serializePendingModelKey() { + expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId) + + key := suite.getPendingModelKey() + serializedKey := key.String() + + assert.Equal(suite.T(), expectedSerializedKey, serializedKey) } func (suite *ModelStorageSuite) TestModelStorage_CompareAndSwap() { From 663540242c108909f4e91fa0db56fbffa7742db9 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Fri, 27 Dec 2024 12:16:41 +0300 Subject: [PATCH 07/22] new methods for train v2 --- authutils/auth_service.go | 6 +- blockchain/orginzationMetadata.go | 4 +- go.mod | 12 +- go.sum | 14 + handler/interceptors.go | 10 + training/mock.go | 2 +- training/service.go | 440 +++++++++++++++++++----------- training/service_test.go | 14 +- training/training_daemon.proto | 3 +- training/training_v2.proto | 13 +- 10 files changed, 336 insertions(+), 182 deletions(-) diff --git a/authutils/auth_service.go b/authutils/auth_service.go index 887935b2..655624b2 100755 --- a/authutils/auth_service.go +++ b/authutils/auth_service.go @@ -65,15 +65,15 @@ func GetSignerAddressFromMessage(message, signature []byte) (signer *common.Addr messageHashFieldLog) return nil, errors.New("incorrect signature data") } - publicKeyFieldLog := zap.Any("publicKey", publicKey) + //publicKeyFieldLog := zap.Any("publicKey", publicKey) keyOwnerAddress := crypto.PubkeyToAddress(*publicKey) keyOwnerAddressFieldLog := zap.Any("keyOwnerAddress", keyOwnerAddress) zap.L().Debug("Message signature parsed", - messageFieldLog, + //messageFieldLog, signatureFieldLog, messageHashFieldLog, - publicKeyFieldLog, + //publicKeyFieldLog, keyOwnerAddressFieldLog) return &keyOwnerAddress, nil diff --git a/blockchain/orginzationMetadata.go b/blockchain/orginzationMetadata.go index 54401ea4..6b12fa54 100644 --- a/blockchain/orginzationMetadata.go +++ b/blockchain/orginzationMetadata.go @@ -184,13 +184,13 @@ func GetOrganizationMetaDataFromIPFS(hash string) (*OrganizationMetaData, error) } func getMetaDataURI() []byte { - //Block chain call here to get the hash of the metadata for the given Organization + // Blockchain call here to get the hash of the metadata for the given Organization reg := getRegistryCaller() orgId := StringToBytes32(config.GetString(config.OrganizationId)) organizationRegistered, err := reg.GetOrganizationById(nil, orgId) if err != nil || !organizationRegistered.Found { - zap.L().Panic("Error Retrieving contract details for the Given Organization", zap.String("OrganizationId", config.GetString(config.OrganizationId)), zap.Error(err)) + zap.L().Panic("Error Retrieving contract details for the Given Organization, recheck blockchain provider endpoint", zap.String("OrganizationId", config.GetString(config.OrganizationId)), zap.Error(err)) } return organizationRegistered.OrgMetadataURI[:] } diff --git a/go.mod b/go.mod index ea9bc45e..97566218 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23 require ( github.com/OneOfOne/go-utils v0.0.0-20180319162427-6019ff89a94e github.com/bufbuild/protocompile v0.14.1 - github.com/emicklei/proto v1.13.3 + github.com/emicklei/proto v1.14.0 github.com/ethereum/go-ethereum v1.14.12 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang-jwt/jwt/v5 v5.2.1 @@ -22,7 +22,7 @@ require ( github.com/rs/xid v1.6.0 github.com/singnet/snet-ecosystem-contracts v0.1.1 github.com/soheilhy/cmux v0.1.5 - github.com/spf13/cast v1.7.0 + github.com/spf13/cast v1.7.1 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 @@ -31,10 +31,10 @@ require ( go.etcd.io/etcd/server/v3 v3.5.17 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/time v0.8.0 - google.golang.org/grpc v1.69.0 - google.golang.org/protobuf v1.35.2 + google.golang.org/grpc v1.69.2 + google.golang.org/protobuf v1.36.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -260,7 +260,7 @@ require ( gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect diff --git a/go.sum b/go.sum index ea046a73..f634adf4 100644 --- a/go.sum +++ b/go.sum @@ -183,6 +183,8 @@ github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/u github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/emicklei/proto v1.13.3 h1:h2u3JgIKSBWHI0jTFFAxiIzOQAe+bRGlQ/WlaIa49Uk= github.com/emicklei/proto v1.13.3/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emicklei/proto v1.14.0 h1:WYxC0OrBuuC+FUCTZvb8+fzEHdZMwLEF+OnVfZA3LXU= +github.com/emicklei/proto v1.14.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -971,6 +973,8 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -1257,6 +1261,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1464,6 +1470,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1: google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb h1:3oy2tynMOP1QbTC0MsNNAV+Se8M2Bd0A5+x1QHyw+pI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1485,6 +1493,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.69.0 h1:quSiOM1GJPmPH5XtU+BCoVXcDVJJAzNcoyfC2cCjGkI= google.golang.org/grpc v1.69.0/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1499,6 +1509,10 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= +google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/handler/interceptors.go b/handler/interceptors.go index 2df91152..650a52a2 100644 --- a/handler/interceptors.go +++ b/handler/interceptors.go @@ -264,8 +264,18 @@ func (interceptor *paymentValidationInterceptor) intercept(srv any, ss grpc.Serv if err != nil { return err.Err() } + zap.L().Debug("New gRPC call received", zap.Any("context", context)) + if context.Info.FullMethod == "/training_daemon.Daemon/upload_and_validate" { + e = handler(srv, wrapperStream) + if e != nil { + zap.L().Warn("gRPC handler returned error", zap.Error(e)) + return e + } + return nil + } + paymentHandler, err := interceptor.getPaymentHandler(context) if err != nil { return err.Err() diff --git a/training/mock.go b/training/mock.go index c8bfba95..0f7a6480 100644 --- a/training/mock.go +++ b/training/mock.go @@ -136,6 +136,6 @@ func (n NoModelSupportService) mustEmbedUnimplementedModelServer() { panic("implement me") } -func (ds DaemonService) mustEmbedUnimplementedDaemonServer() { +func (ds *DaemonService) mustEmbedUnimplementedDaemonServer() { panic("implement me") } diff --git a/training/service.go b/training/service.go index b806a793..dd0c7cb4 100644 --- a/training/service.go +++ b/training/service.go @@ -1,11 +1,13 @@ -//go:generate protoc -I . ./training_v2.proto --go-grpc_out=. --go_out=. -//go:generate protoc -I . ./training_daemon.proto --go-grpc_out=. --go_out=. +//go:generate protoc -I . ./training_daemon.proto ./training_v2.proto --go-grpc_out=. --go_out=. package training import ( + "bytes" "context" + "errors" "fmt" + "io" "maps" "net/url" "slices" @@ -39,9 +41,6 @@ const ( //go:embed training_v2.proto var TrainingProtoEmbeded string -//type IService interface { -//} - // ModelService this is remote AI service provider type ModelService struct { serviceMetaData *blockchain.ServiceMetadata @@ -87,15 +86,17 @@ func NewDaemonsService( func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest) (*ModelResponse, error) { + zap.L().Debug("CreateModel request") + if request == nil || request.Authorization == nil { return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("invalid request, no authorization provided") } if err := ds.verifySignature(request.Authorization); err != nil { - zap.L().Error("unable to create model", zap.Error(err)) + zap.L().Error("unable to create model, bad authorization provided", zap.Error(err)) return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("unable to create model: %v", err) + fmt.Errorf("bad authorization provided: %v", err) } if request.GetModel().GrpcServiceName == "" || request.GetModel().GrpcMethodName == "" { @@ -106,40 +107,44 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest request.Model.ServiceId = config.GetString(config.ServiceId) request.Model.OrganizationId = config.GetString(config.OrganizationId) - request.Model.GroupId = config.GetString(config.DaemonGroupName) + request.Model.GroupId = ds.organizationMetaData.GetGroupIdString() // make a call to the client // if the response is successful, store details in etcd // send back the response to the client conn, client, err := ds.getServiceClient() - deferConnection(conn) if err != nil { + zap.L().Error("[CreateModel] unable to getServiceClient", zap.Error(err)) return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for Model Training %v", err) } responseModelID, errClient := client.CreateModel(c, request.Model) + closeConn(conn) if errClient != nil { + zap.L().Error("[CreateModel] unable to call CreateModel", zap.Error(errClient)) return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for Model Training %v", errClient) } if responseModelID == nil { + zap.L().Error("[CreateModel] CreateModel returned null response") return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for model training: service return empty response") } if responseModelID.ModelId == "" { + zap.L().Error("[CreateModel] CreateModel returned empty modelID") return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for model training: service return empty modelID") } //store the details in etcd - zap.L().Info("Creating model based on response from CreateModel of training service") + zap.L().Debug("Creating model based on response from CreateModel of training service") data, err := ds.createModelDetails(request, responseModelID) if err != nil { - zap.L().Error(err.Error()) + zap.L().Error("[CreateModel] Can't save model", zap.Error(err)) return nil, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) } modelResponse := BuildModelResponse(data, Status_CREATED) @@ -152,9 +157,7 @@ func (ds *DaemonService) getTrainingAndValidatingModelIds(orgId, serviceId, grou } func (ds *DaemonService) startUpdateModelStatusWorker(ctx context.Context, modelId string) { - modelKey := ds.buildModelKey(modelId) - currentModelData, ok, err := ds.storage.Get(modelKey) if err != nil { zap.L().Error("err in getting modelData from storage", zap.Error(err)) @@ -223,23 +226,132 @@ func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, int } func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { - panic("implement me") + conn, client, err := ds.getServiceClient() + if client == nil || err != nil { + return &PriceInBaseUnit{ + Price: 0, + }, fmt.Errorf("issue with service: %v", err) + } + price, err := client.ValidateModelPrice(context.Background(), &ValidateRequest{ + ModelId: request.ModelId, + TrainingDataLink: request.TrainingDataLink, + }) + closeConn(conn) + if err != nil || price == nil { + zap.L().Error("issue with ValidateModelPrice", zap.Error(err)) + return nil, fmt.Errorf("issue with service: %v", err) + } + return price, nil } -func (ds *DaemonService) UploadAndValidate(server Daemon_UploadAndValidateServer) error { - panic("implement me") +// TODO fix +func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer) error { + var fullData bytes.Buffer // Для хранения всего принятого файла + var modelID string + + conn, client, err := ds.getServiceClient() + if err != nil { + zap.L().Debug(err.Error()) + return err + } + //deferConnection(conn) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*55) + defer cancel() + grpcStream, err := client.UploadAndValidate(ctx) + if err != nil { + zap.L().Error("error in sending UploadAndValidate", zap.Error(err)) + return err + } + + zap.L().Debug("UploadAndValidate CALLED") + for { + // Получаем сообщение из потока + req, err := stream.Recv() + if req != nil { + zap.L().Debug("stream.Recv() for model_id " + req.UploadInput.ModelId) + } + if err == io.EOF { + // Клиент завершил отправку данных + zap.L().Debug("Клиент завершил отправку данных") + break + } + if err != nil { + zap.L().Debug("req is nil?", zap.Bool("bool", req == nil)) + zap.L().Error("error in receiving upload request", zap.Error(err)) + return err + } + + err = grpcStream.SendMsg(req.UploadInput) + if err != nil { + zap.L().Error("error in sending upload validation response", zap.Error(err)) + return err + } + + // Обрабатываем данные + zap.L().Debug("Received chunk of data for model", zap.String("modelID", modelID)) + fullData.Write(req.UploadInput.Data) // Добавляем данные в общий буфер + } + zap.L().Debug("Received file for model %s with size %d bytes", zap.String("modelID", modelID), zap.Int("len", fullData.Len())) + closeConn(conn) + return stream.SendAndClose(&StatusResponse{ + Status: Status_VALIDATING, + }) } func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidateRequest) (*StatusResponse, error) { - panic("implement me") + conn, client, err := ds.getServiceClient() + if client == nil || err != nil { + return &StatusResponse{ + Status: Status_ERRORED, + }, fmt.Errorf("issue with service: %v", err) + } + price, err := client.ValidateModel(ctx, &ValidateRequest{ + ModelId: request.ModelId, + TrainingDataLink: request.TrainingDataLink, + }) + closeConn(conn) + if err != nil { + return nil, fmt.Errorf("[ValidateModel] issue with service: %v", err) + } + return price, nil } func (ds *DaemonService) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { - panic("implement me") + conn, client, err := ds.getServiceClient() + if client == nil || err != nil { + return &PriceInBaseUnit{ + Price: 0, + }, fmt.Errorf("issue with service: %v", err) + } + price, err := client.TrainModelPrice(ctx, &ModelID{ + ModelId: request.ModelId, + }) + closeConn(conn) + if err != nil { + return nil, fmt.Errorf("issue with service: %v", err) + } + return price, nil } func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { - panic("implement me") + conn, client, err := ds.getServiceClient() + if client == nil || err != nil { + zap.L().Error("issue with service", zap.Error(err)) + return &StatusResponse{ + Status: Status_ERRORED, + }, fmt.Errorf("issue with service: %v", err) + } + statusResp, err := client.TrainModel(ctx, &ModelID{ + ModelId: request.ModelId, + }) + closeConn(conn) + if err != nil { + zap.L().Error("[TrainModel] issue with service", zap.Error(err)) + return &StatusResponse{ + Status: Status_ERRORED, + }, fmt.Errorf("issue with service: %v", err) + } + return statusResp, nil } func (ds *DaemonService) GetTrainingMetadata(ctx context.Context, empty *emptypb.Empty) (*TrainingMetadata, error) { @@ -247,7 +359,30 @@ func (ds *DaemonService) GetTrainingMetadata(ctx context.Context, empty *emptypb } func (ds *DaemonService) UpdateModel(ctx context.Context, request *UpdateModelRequest) (*ModelResponse, error) { - panic("implement me") + + if request == nil || request.Authorization == nil { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("invalid request, no authorization provided") + } + if err := ds.verifySignature(request.Authorization); err != nil { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("unable to access model: %v", err) + } + if err := ds.verifySignerHasAccessToTheModel(request.ModelId, request.Authorization.SignerAddress); err != nil { + return &ModelResponse{}, + fmt.Errorf("unable to access model: %v", err) + } + if request.ModelId == "" { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("invalid request, no model id provided") + } + + zap.L().Info("Updating model") + data, err := ds.updateModelDetails(request) + if err != nil || data == nil { + return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) + } + return BuildModelResponse(data, data.Status), nil } func (ds *DaemonService) GetMethodMetadata(ctx context.Context, request *MethodMetadataRequest) (*MethodMetadata, error) { @@ -258,15 +393,31 @@ func (ds *DaemonService) GetMethodMetadata(ctx context.Context, request *MethodM return ds.methodsMetadata[key], nil } -func deferConnection(conn *grpc.ClientConn) { - defer func(conn *grpc.ClientConn) { - err := conn.Close() - if err != nil { - zap.L().Error("error in closing Client Connection", zap.Error(err)) - } - }(conn) +func closeConn(conn *grpc.ClientConn) { + if conn == nil { + zap.L().Debug("conn is nil!") + return + } + err := conn.Close() + if err != nil { + zap.L().Error("error in closing Client Connection", zap.Error(err)) + } } +// deprecated +//func deferConnection(conn *grpc.ClientConn) { +// if conn == nil { +// zap.L().Debug("conn is nil!") +// return +// } +// defer func(conn *grpc.ClientConn) { +// err := conn.Close() +// if err != nil { +// zap.L().Error("error in closing Client Connection", zap.Error(err)) +// } +// }(conn) +//} + func getConnection(endpoint string) (conn *grpc.ClientConn) { options := grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(config.GetInt(config.MaxMessageSizeInMB)*1024*1024), @@ -307,7 +458,7 @@ func (ds *DaemonService) createModelDetails(request *NewModelRequest, response * zap.L().Error("can't put model in etcd", zap.Error(err)) return } - //for every accessible address in the list, store the user address and all the model Ids associated with it + // for every accessible address in the list, store the user address and all the model Ids associated with it for _, address := range data.AuthorizedAddresses { userKey := getModelUserKey(key, address) userData := ds.getModelUserData(key, address) @@ -343,7 +494,7 @@ func (ds *DaemonService) getModelUserData(key *ModelKey, address string) *ModelU modelIds = data.ModelIds } if err != nil { - zap.L().Error("can't get model data from etcd", zap.Error(err)) + zap.L().Error("[getModelUserData] can't get model data from etcd", zap.Error(err)) } modelIds = append(modelIds, key.ModelId) return &ModelUserData{ @@ -357,15 +508,18 @@ func (ds *DaemonService) getModelUserData(key *ModelKey, address string) *ModelU } } -func (ds DaemonService) deleteUserModelDetails(key *ModelKey, data *ModelData) (err error) { +func (ds *DaemonService) deleteUserModelDetails(key *ModelKey, data *ModelData) (err error) { for _, address := range data.AuthorizedAddresses { userKey := getModelUserKey(key, address) - if data, ok, err := ds.userStorage.Get(userKey); ok && err == nil && data != nil { - data.ModelIds = remove(data.ModelIds, key.ModelId) - err = ds.userStorage.Put(userKey, data) - if err != nil { - zap.L().Error(err.Error()) - } + dataStorage, ok, err := ds.userStorage.Get(userKey) + if !ok || err != nil || dataStorage == nil { + zap.L().Error("[deleteUserModelDetails] can't get user data", zap.Error(err)) + continue + } + dataStorage.ModelIds = remove(dataStorage.ModelIds, key.ModelId) + err = ds.userStorage.Put(userKey, dataStorage) + if err != nil { + zap.L().Error("can't remove access to model", zap.Error(err), zap.String("userKey", userKey.String()), zap.String("modelID", key.ModelId)) } } return @@ -375,12 +529,17 @@ func (ds *DaemonService) deleteModelDetails(req *CommonRequest) (data *ModelData key := ds.getModelKeyToUpdate(req.ModelId) ok := false data, ok, err = ds.storage.Get(key) - if ok && err == nil { - data.Status = Status_DELETED - data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) - err = ds.storage.Put(key, data) - err = ds.deleteUserModelDetails(key, data) + if data == nil || !ok || err != nil { + zap.L().Debug("Can't find model to delete", zap.String("key", key.String())) + return nil, errors.New("can't find model to delete") } + data.Status = Status_DELETED + data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) + err = ds.storage.Put(key, data) + if err != nil { + zap.L().Error("can't update status to DELETED", zap.Error(err)) + } + err = ds.deleteUserModelDetails(key, data) return } @@ -403,55 +562,55 @@ func convertModelDataToBO(data *ModelData) (responseData *ModelResponse) { return } -//func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest, response *ModelDetailsResponse) (data *ModelData, err error) { -// key := ds.getModelKeyToUpdate(request.Mo) -// oldAddresses := make([]string, 0) -// var latestAddresses []string -// // by default add the creator to the Authorized list of Address -// if request.UpdateModelDetails.AddressList != nil || len(request.UpdateModelDetails.AddressList) > 0 { -// latestAddresses = request.UpdateModelDetails.AddressList -// } -// latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) -// if data, err = ds.getModelDataForUpdate(request); err == nil && data != nil { -// oldAddresses = data.AuthorizedAddresses -// data.AuthorizedAddresses = latestAddresses -// latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) -// data.IsPublic = request.UpdateModelDetails.IsPubliclyAccessible -// data.UpdatedByAddress = request.Authorization.SignerAddress -// if response != nil { -// data.Status = response.Status -// } -// data.ModelName = request.UpdateModelDetails.ModelName -// data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) -// data.Description = request.UpdateModelDetails.Description -// -// err = ds.storage.Put(key, data) -// if err != nil { -// zap.L().Error("Error in putting data in user storage", zap.Error(err)) -// } -// //get the difference of all the addresses b/w old and new -// updatedAddresses := difference(oldAddresses, latestAddresses) -// for _, address := range updatedAddresses { -// modelUserKey := getModelUserKey(key, address) -// modelUserData := service.getModelUserData(key, address) -// //if the address is present in the request but not in the old address , add it to the storage -// if isValuePresent(address, request.UpdateModelDetails.AddressList) { -// modelUserData.ModelIds = append(modelUserData.ModelIds, request.UpdateModelDetails.ModelId) -// } else { // the address was present in the old data , but not in new , hence needs to be deleted -// modelUserData.ModelIds = remove(modelUserData.ModelIds, request.UpdateModelDetails.ModelId) -// } -// err = ds.userStorage.Put(modelUserKey, modelUserData) -// if err != nil { -// zap.L().Error("Error in putting data in storage", zap.Error(err)) -// } -// } -// -// } -// return -//} +func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data *ModelData, err error) { + key := ds.getModelKeyToUpdate(request.ModelId) + oldAddresses := make([]string, 0) + var latestAddresses []string + // by default add the creator to the Authorized list of Address + if request.AddressList != nil || len(request.AddressList) > 0 { + latestAddresses = request.AddressList + } + latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) // add creator + if data, err = ds.getModelDataForUpdate(request.ModelId); err == nil && data != nil { + oldAddresses = data.AuthorizedAddresses + data.AuthorizedAddresses = latestAddresses + latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) + // TODO data.IsPublic = request.IsIsPubliclyAccessible + data.UpdatedByAddress = request.Authorization.SignerAddress + //if response != nil { + // data.Status = response.Status + //} + data.ModelName = request.ModelName + data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) + data.Description = request.Description + + err = ds.storage.Put(key, data) + if err != nil { + zap.L().Error("Error in putting data in user storage", zap.Error(err)) + } + //get the difference of all the addresses b/w old and new + updatedAddresses := difference(oldAddresses, latestAddresses) + for _, address := range updatedAddresses { + modelUserKey := getModelUserKey(key, address) + modelUserData := ds.getModelUserData(key, address) + //if the address is present in the request but not in the old address , add it to the storage + if isValuePresent(address, request.AddressList) { + modelUserData.ModelIds = append(modelUserData.ModelIds, request.ModelId) + } else { // the address was present in the old data , but not in new , hence needs to be deleted + modelUserData.ModelIds = remove(modelUserData.ModelIds, request.ModelId) + } + err = ds.userStorage.Put(modelUserKey, modelUserData) + if err != nil { + zap.L().Error("Error in putting data in storage", zap.Error(err)) + } + } + + } + return +} // ensure only authorized use -func (ds DaemonService) verifySignerHasAccessToTheModel(modelId string, address string) (err error) { +func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address string) (err error) { key := &ModelUserKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), @@ -469,24 +628,25 @@ func (ds DaemonService) verifySignerHasAccessToTheModel(modelId string, address return } -func (ds DaemonService) updateModelDetailsWithLatestStatus(request *CommonRequest, response *ModelResponse) (data *ModelData, err error) { +func (ds *DaemonService) updateModelStatus(request *CommonRequest, newStatus Status) (data *ModelData, err error) { key := &ModelKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.ModelDetails.GrpcMethodName, - //GRPCServiceName: request.ModelDetails.GrpcServiceName, - ModelId: request.ModelId, + ModelId: request.ModelId, } ok := false - zap.L().Debug("updateModelDetailsWithLatestStatus: ", zap.Any("key", key)) - if data, ok, err = ds.storage.Get(key); err == nil && ok { - data.Status = response.Status - if err = ds.storage.Put(key, data); err != nil { - zap.L().Error("issue with retrieving model data from storage", zap.Error(err)) - } + zap.L().Debug("[updateModelStatus]", zap.String("modelID", key.ModelId)) + data, ok, err = ds.storage.Get(key) + if err != nil || !ok || data == nil { + zap.L().Error("[updateModelStatus] can't get model data from etcd", zap.Error(err)) + return data, errors.New("can't get model data from etcd") + } + data.Status = newStatus + if err = ds.storage.Put(key, data); err != nil { + zap.L().Error("[updateModelStatus] issue with retrieving model data from storage", zap.Error(err)) + return data, fmt.Errorf("can't get model data from etcd: %s", err) } - zap.L().Error("can't get model data from etcd", zap.Error(err)) return } @@ -514,24 +674,24 @@ func (ds *DaemonService) getModelKeyToUpdate(modelID string) (key *ModelKey) { return } -//func (ds *DaemonService) getModelDataForUpdate(request *UpdateModelRequest) (data *ModelData, err error) { -// key := ds.getModelKeyToUpdate(request) -// ok := false -// -// if data, ok, err = ds.storage.Get(key); err != nil || !ok { -// zap.L().Warn("unable to retrieve model data from storage", zap.String("Model Id", key.ModelId), zap.Error(err)) -// } -// return -//} +func (ds *DaemonService) getModelDataForUpdate(modelID string) (data *ModelData, err error) { + key := ds.getModelKeyToUpdate(modelID) + ok := false + + if data, ok, err = ds.storage.Get(key); err != nil || !ok { + zap.L().Warn("unable to retrieve model data from storage", zap.String("Model Id", key.ModelId), zap.Error(err)) + } + return +} func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (response *ModelsResponse, err error) { if request == nil || request.Authorization == nil { return &ModelsResponse{}, - fmt.Errorf("Invalid request , no Authorization provided ") + fmt.Errorf("invalid request, no Authorization provided ") } if err = ds.verifySignature(request.Authorization); err != nil { return &ModelsResponse{}, - fmt.Errorf("Unable to access model, %v", err) + fmt.Errorf("unable to access model: %v", err) } //if request.GetGrpcMethodName() == "" || request.GetGrpcServiceName() == "" { // return &AccessibleModelsResponse{}, @@ -575,7 +735,7 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque return } -func (ds DaemonService) getModelDataToCreate(request *NewModelRequest, response *ModelID) (data *ModelData) { +func (ds *DaemonService) getModelDataToCreate(request *NewModelRequest, response *ModelID) (data *ModelData) { data = &ModelData{ Status: Status_CREATED, GRPCServiceName: request.Model.GrpcServiceName, @@ -585,7 +745,7 @@ func (ds DaemonService) getModelDataToCreate(request *NewModelRequest, response AuthorizedAddresses: request.Model.AddressList, Description: request.Model.Description, ModelName: request.Model.Name, - //TrainingLink: + //TrainingLink: TODO IsPublic: request.Model.IsPublic, IsDefault: false, ModelId: response.ModelId, @@ -613,45 +773,10 @@ func BuildModelResponse(data *ModelData, status Status) *ModelResponse { AddressList: data.AuthorizedAddresses, TrainingDataLink: data.TrainingLink, Name: data.ModelName, - //OrganizationId: config.GetString(config.OrganizationId), - //ServiceId: config.GetString(config.ServiceId), - //GroupId: data.GroupId, - //Status: status.String(), - UpdatedDate: data.UpdatedDate, + UpdatedDate: data.UpdatedDate, } } -//func (ds *DaemonService) UpdateModelAccess(c context.Context, request *UpdateModelRequest) (response *ModelDetailsResponse, -// err error) { -// if request == nil || request.Authorization == nil { -// return &ModelDetailsResponse{Status: Status_ERRORED}, -// fmt.Errorf(" Invalid request , no Authorization provided , %v", err) -// } -// if err = ds.verifySignature(request.Authorization); err != nil { -// return &ModelDetailsResponse{Status: Status_ERRORED}, -// fmt.Errorf(" Unable to access model , %v", err) -// } -// if err = service.verifySignerHasAccessToTheModel(request.UpdateModelDetails.GrpcServiceName, -// request.UpdateModelDetails.GrpcMethodName, request.UpdateModelDetails.ModelId, request.Authorization.SignerAddress); err != nil { -// return &ModelDetailsResponse{}, -// fmt.Errorf(" Unable to access model , %v", err) -// } -// if request.UpdateModelDetails == nil { -// return &ModelDetailsResponse{Status: Status_ERRORED}, -// fmt.Errorf(" Invalid request , no UpdateModelDetails provided , %v", err) -// } -// -// zap.L().Info("Updating model based on response from UpdateModel") -// if data, err := service.updateModelDetails(request, response); err == nil && data != nil { -// response = BuildModelResponse(data, data.Status) -// -// } else { -// return response, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) -// } -// -// return -//} - func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*StatusResponse, error) { if req == nil || req.Authorization == nil { @@ -674,20 +799,20 @@ func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*St fmt.Errorf("unable to access model , %v", err) } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) defer cancel() conn, client, err := ds.getServiceClient() if err != nil { return &StatusResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for Model Training") } - deferConnection(conn) - response, err := client.DeleteModel(ctx, &ModelID{ModelId: req.ModelId}) - if response == nil || err != nil { - zap.L().Error("error in invoking DeleteModel, service-provider should realize it", zap.Error(err)) + response, errModel := client.DeleteModel(ctx, &ModelID{ModelId: req.ModelId}) + closeConn(conn) + if response == nil || errModel != nil { + zap.L().Error("error in invoking DeleteModel, service-provider should realize it", zap.Error(errModel)) return &StatusResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking DeleteModel, service-provider should realize it") } data, err := ds.deleteModelDetails(req) - if err == nil && data != nil { + if err != nil || data == nil { zap.L().Error("issue with deleting ModelId in storage", zap.Error(err)) return response, fmt.Errorf("issue with deleting Model Id in Storage %v", err) } @@ -723,9 +848,10 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re zap.L().Error("error in invoking GetModelStatus, service-provider should realize it", zap.Error(err)) return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking GetModelStatus, service-provider should realize it") } - zap.L().Info("[GetModelStatus] response from service-provider", zap.Any("response", response)) + zap.L().Info("[GetModelStatus] response from service-provider", zap.Any("response", responseStatus)) zap.L().Info("[GetModelStatus] updating model status based on response from UpdateModel") - data, err := ds.updateModelDetailsWithLatestStatus(request, response) + data, err := ds.updateModelStatus(request, responseStatus.Status) + closeConn(conn) zap.L().Debug("[GetModelStatus] data that be returned to client", zap.Any("data", data)) if err == nil && data != nil { response = BuildModelResponse(data, responseStatus.Status) @@ -733,7 +859,6 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re zap.L().Error("[GetModelStatus] BuildModelResponse error", zap.Error(err)) return response, fmt.Errorf("[GetModelStatus] issue with storing Model Id in the Daemon Storage %v", err) } - deferConnection(conn) } else { return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("[GetModelStatus] error in invoking service for Model Training") } @@ -817,6 +942,7 @@ func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMet // if the method accepts modelId, then we consider it as training if inputFields.Get(fieldsCounter).Message().FullName() == "trainingV2.ModelID" { // init array if nil + trainingMD.TrainingInProto = true if trainingMD.TrainingMethods[string(protoService.Name())] == nil { trainingMD.TrainingMethods[string(protoService.Name())], _ = structpb.NewList(nil) } diff --git a/training/service_test.go b/training/service_test.go index 0d20153d..a2727883 100644 --- a/training/service_test.go +++ b/training/service_test.go @@ -224,12 +224,12 @@ func (suite *ModelServiceTestSuite) TestModelService_CreateModel() { //check if the model Id stored has all the details key := &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - GRPCMethodName: "/example_service.Calculator/train_add", - GRPCServiceName: " ", - ModelId: "1", + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), + //GRPCMethodName: "/example_service.Calculator/train_add", + //GRPCServiceName: " ", + ModelId: "1", } modelData, ok, err := suite.service.(*ModelService).storage.Get(key) assert.Equal(suite.T(), []string{"A1", "A2", "A3", suite.senderAddress.String()}, modelData.AuthorizedAddresses) @@ -411,7 +411,7 @@ func (suite *ModelServiceTestSuite) TestModelService_UpdateModelAccess() { } -func (suite *ModelServiceTestSuite) TestModelService_GetAllAccessibleModels() { +func (suite *ModelServiceTestSuite) TestModelService_GetAllModels() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*2000) defer cancel() request2 := &AccessibleModelsRequest{ diff --git a/training/training_daemon.proto b/training/training_daemon.proto index b130baa3..e86ff308 100644 --- a/training/training_daemon.proto +++ b/training/training_daemon.proto @@ -29,8 +29,7 @@ message AuthValidateRequest { message UploadAndValidateRequest { AuthorizationDetails authorization = 1; - string model_id = 2; - bytes data = 3; + trainingV2.UploadInput upload_input = 2; } message CommonRequest { diff --git a/training/training_v2.proto b/training/training_v2.proto index f6e778fa..66007a27 100644 --- a/training/training_v2.proto +++ b/training/training_v2.proto @@ -14,7 +14,7 @@ service Model { rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {} // Paid - rpc upload_and_validate(stream DataToUpload) returns (StatusResponse) {} + rpc upload_and_validate(stream UploadInput) returns (StatusResponse) {} // Paid rpc validate_model(ValidateRequest) returns (StatusResponse) {} @@ -97,9 +97,14 @@ message StatusResponse { Status status = 1; } -message DataToUpload { - string model_id = 2; - bytes data = 3; +message UploadInput { + string model_id = 1; + bytes data = 2; + string file_name = 3; + uint64 file_size = 4; // in bytes + uint64 batch_size = 5; + uint64 batch_number = 6; + uint64 batch_count = 7; } message ValidateRequest { From fed47afc4a118ebf4ac32dbc7a6420df72316b33 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Fri, 27 Dec 2024 15:47:05 +0300 Subject: [PATCH 08/22] fix pending storage --- snetd/cmd/components.go | 11 ++++++++++- training/service.go | 3 ++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index fa2b10f5..b08c4e00 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -63,6 +63,7 @@ type Components struct { trainingService training.DaemonServer modelUserStorage *training.ModelUserStorage modelStorage *training.ModelStorage + pendingModelStorage *training.PendingModelStorage } func InitComponents(cmd *cobra.Command) (components *Components) { @@ -568,6 +569,14 @@ func (components *Components) ModelUserStorage() *training.ModelUserStorage { return components.modelUserStorage } +func (components *Components) PendingModelStorage() *training.PendingModelStorage { + if components.pendingModelStorage != nil { + return components.pendingModelStorage + } + components.pendingModelStorage = training.NewPendingModelStorage(components.AtomicStorage()) + return components.pendingModelStorage +} + func (components *Components) ModelStorage() *training.ModelStorage { if components.modelStorage != nil { return components.modelStorage @@ -586,7 +595,7 @@ func (components *Components) TrainingService() training.DaemonServer { } components.trainingService = training.NewTrainingService(components.PaymentChannelService(), components.ServiceMetaData(), - components.OrganizationMetaData(), components.ModelStorage(), components.ModelUserStorage()) + components.OrganizationMetaData(), components.ModelStorage(), components.ModelUserStorage(), components.PendingModelStorage()) return components.trainingService } diff --git a/training/service.go b/training/service.go index 78631528..d44cde84 100644 --- a/training/service.go +++ b/training/service.go @@ -920,7 +920,7 @@ func NewModelService(channelService escrow.PaymentChannelService, serMetaData *b // NewTrainingService daemon self server func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData *blockchain.ServiceMetadata, - orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage) DaemonServer { + orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage, pendingStorage *PendingModelStorage) DaemonServer { linkerFiles := getFileDescriptors(serMetaData.ProtoFiles) serMetaData.ProtoDescriptors = linkerFiles @@ -939,6 +939,7 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData organizationMetaData: orgMetadata, storage: storage, userStorage: userStorage, + pendingStorage: pendingStorage, serviceUrl: serviceURL, trainingMetadata: &trainMD, methodsMetadata: methodsMD, From ce408c9bf90c92f2698caf888c09eee60d61c13e Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Fri, 27 Dec 2024 16:41:12 +0300 Subject: [PATCH 09/22] refactor trainv2 --- training/mock.go | 16 +++++++-------- training/service.go | 49 +++++++++++++++------------------------------ 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/training/mock.go b/training/mock.go index 0f7a6480..7b05dc3a 100644 --- a/training/mock.go +++ b/training/mock.go @@ -101,35 +101,35 @@ func (service ModelService) mustEmbedUnimplementedModelServer() { } func (n NoModelSupportService) CreateModel(ctx context.Context, model *NewModel) (*ModelID, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) ValidateModelPrice(ctx context.Context, request *ValidateRequest) (*PriceInBaseUnit, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) UploadAndValidate(server Model_UploadAndValidateServer) error { - return fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) ValidateModel(ctx context.Context, request *ValidateRequest) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) TrainModelPrice(ctx context.Context, id *ModelID) (*PriceInBaseUnit, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) TrainModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) DeleteModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) GetModelStatus(ctx context.Context, id *ModelID) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") + return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } func (n NoModelSupportService) mustEmbedUnimplementedModelServer() { diff --git a/training/service.go b/training/service.go index d44cde84..cb0862a0 100644 --- a/training/service.go +++ b/training/service.go @@ -281,7 +281,7 @@ func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthVa // TODO fix func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer) error { - var fullData bytes.Buffer // Для хранения всего принятого файла + var fullData bytes.Buffer var modelID string conn, client, err := ds.getServiceClient() @@ -289,7 +289,6 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer zap.L().Debug(err.Error()) return err } - //deferConnection(conn) ctx, cancel := context.WithTimeout(context.Background(), time.Second*55) defer cancel() grpcStream, err := client.UploadAndValidate(ctx) @@ -300,14 +299,12 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer zap.L().Debug("UploadAndValidate CALLED") for { - // Получаем сообщение из потока req, err := stream.Recv() if req != nil { zap.L().Debug("stream.Recv() for model_id " + req.UploadInput.ModelId) } if err == io.EOF { - // Клиент завершил отправку данных - zap.L().Debug("Клиент завершил отправку данных") + zap.L().Debug("[UploadAndValidate] EOF") break } if err != nil { @@ -322,9 +319,8 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer return err } - // Обрабатываем данные zap.L().Debug("Received chunk of data for model", zap.String("modelID", modelID)) - fullData.Write(req.UploadInput.Data) // Добавляем данные в общий буфер + fullData.Write(req.UploadInput.Data) } zap.L().Debug("Received file for model %s with size %d bytes", zap.String("modelID", modelID), zap.Int("len", fullData.Len())) closeConn(conn) @@ -422,7 +418,13 @@ func (ds *DaemonService) UpdateModel(ctx context.Context, request *UpdateModelRe func (ds *DaemonService) GetMethodMetadata(ctx context.Context, request *MethodMetadataRequest) (*MethodMetadata, error) { if request.GetModelId() != "" { - // TODO get from etcd + data, err := ds.getModelData(request.ModelId) + if err != nil { + zap.L().Error("[GetMethodMetadata] can't get model data", zap.Error(err)) + return nil, fmt.Errorf(" can't get model data: %v", err) + } + request.GrpcMethodName = data.GRPCMethodName + request.GrpcServiceName = data.GRPCServiceName } key := request.GrpcServiceName + request.GrpcMethodName return ds.methodsMetadata[key], nil @@ -561,7 +563,7 @@ func (ds *DaemonService) deleteUserModelDetails(key *ModelKey, data *ModelData) } func (ds *DaemonService) deleteModelDetails(req *CommonRequest) (data *ModelData, err error) { - key := ds.getModelKeyToUpdate(req.ModelId) + key := ds.getModelKey(req.ModelId) ok := false data, ok, err = ds.storage.Get(key) if data == nil || !ok || err != nil { @@ -598,7 +600,7 @@ func convertModelDataToBO(data *ModelData) (responseData *ModelResponse) { } func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data *ModelData, err error) { - key := ds.getModelKeyToUpdate(request.ModelId) + key := ds.getModelKey(request.ModelId) oldAddresses := make([]string, 0) var latestAddresses []string // by default add the creator to the Authorized list of Address @@ -606,7 +608,7 @@ func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data * latestAddresses = request.AddressList } latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) // add creator - if data, err = ds.getModelDataForUpdate(request.ModelId); err == nil && data != nil { + if data, err = ds.getModelData(request.ModelId); err == nil && data != nil { oldAddresses = data.AuthorizedAddresses data.AuthorizedAddresses = latestAddresses latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) @@ -697,7 +699,7 @@ func (ds *DaemonService) buildModelKey(modelID string) (key *ModelKey) { return } -func (ds *DaemonService) getModelKeyToUpdate(modelID string) (key *ModelKey) { +func (ds *DaemonService) getModelKey(modelID string) (key *ModelKey) { key = &ModelKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), @@ -709,10 +711,9 @@ func (ds *DaemonService) getModelKeyToUpdate(modelID string) (key *ModelKey) { return } -func (ds *DaemonService) getModelDataForUpdate(modelID string) (data *ModelData, err error) { - key := ds.getModelKeyToUpdate(modelID) +func (ds *DaemonService) getModelData(modelID string) (data *ModelData, err error) { + key := ds.getModelKey(modelID) ok := false - if data, ok, err = ds.storage.Get(key); err != nil || !ok { zap.L().Warn("unable to retrieve model data from storage", zap.String("Model Id", key.ModelId), zap.Error(err)) } @@ -900,24 +901,6 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re return } -// NewModelService AI service prodiver -// TODO maybe remove -func NewModelService(channelService escrow.PaymentChannelService, serMetaData *blockchain.ServiceMetadata, - orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage) ModelServer { - serviceURL := config.GetString(config.ModelMaintenanceEndPoint) - if config.IsValidUrl(serviceURL) && config.GetBool(config.BlockchainEnabledKey) { - return &ModelService{ - channelService: channelService, - serviceMetaData: serMetaData, - organizationMetaData: orgMetadata, - storage: storage, - userStorage: userStorage, - serviceUrl: serviceURL, - } - } - return &NoModelSupportService{} -} - // NewTrainingService daemon self server func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData *blockchain.ServiceMetadata, orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage, pendingStorage *PendingModelStorage) DaemonServer { From 8c90f1fb71f06852b78d3c6612825181827f6f93 Mon Sep 17 00:00:00 2001 From: Elmir Iskhakov <86847688+elmiringos@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:42:40 +0300 Subject: [PATCH 10/22] Added public model storage for public models (#12) * feat(training): add public model storage * test(training): add unit test for public model storage * feat(training/storage): add methods to update publicModelIds and pendingModelIds * test(training/storage): add TestPendingModelStorage_AddPendingModelId and TestPublicModelStorage_AddPublicModelId * feat(training/service): add public models handling in verifySignerHasAccessToTheModel, update createModelDetails * test(training): add usage newTrainingService instead of NewDaemonService --- snetd/cmd/components.go | 35 +++- training/service.go | 123 +++++++----- training/storage.go | 187 ++++++++++++++++++ .../tests/integration/daemon_service_test.go | 23 ++- training/tests/unit/storage_test.go | 82 +++++++- 5 files changed, 380 insertions(+), 70 deletions(-) diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index b08c4e00..fb4920d1 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -64,6 +64,7 @@ type Components struct { modelUserStorage *training.ModelUserStorage modelStorage *training.ModelStorage pendingModelStorage *training.PendingModelStorage + publicModelStorage *training.PublicModelStorage } func InitComponents(cmd *cobra.Command) (components *Components) { @@ -561,11 +562,23 @@ func (components *Components) ConfigurationService() *configuration_service.Conf return components.configurationService } +func (components *Components) ModelStorage() *training.ModelStorage { + if components.modelStorage != nil { + return components.modelStorage + } + + components.modelStorage = training.NewModelStorage(components.AtomicStorage()) + + return components.modelStorage +} + func (components *Components) ModelUserStorage() *training.ModelUserStorage { if components.modelUserStorage != nil { return components.modelUserStorage } + components.modelUserStorage = training.NewUserModelStorage(components.AtomicStorage()) + return components.modelUserStorage } @@ -573,16 +586,20 @@ func (components *Components) PendingModelStorage() *training.PendingModelStorag if components.pendingModelStorage != nil { return components.pendingModelStorage } + components.pendingModelStorage = training.NewPendingModelStorage(components.AtomicStorage()) + return components.pendingModelStorage } -func (components *Components) ModelStorage() *training.ModelStorage { - if components.modelStorage != nil { - return components.modelStorage +func (components *Components) PublicModelStorage() *training.PublicModelStorage { + if components.publicModelStorage != nil { + return components.publicModelStorage } - components.modelStorage = training.NewModelStorage(components.AtomicStorage()) - return components.modelStorage + + components.publicModelStorage = training.NewPublicModelStorage(components.AtomicStorage()) + + return components.publicModelStorage } func (components *Components) TrainingService() training.DaemonServer { @@ -594,8 +611,12 @@ func (components *Components) TrainingService() training.DaemonServer { return components.trainingService } - components.trainingService = training.NewTrainingService(components.PaymentChannelService(), components.ServiceMetaData(), - components.OrganizationMetaData(), components.ModelStorage(), components.ModelUserStorage(), components.PendingModelStorage()) + components.trainingService = training.NewTrainingService( + components.PaymentChannelService(), components.ServiceMetaData(), + components.OrganizationMetaData(), components.ModelStorage(), + components.ModelUserStorage(), components.PendingModelStorage(), + components.PublicModelStorage()) + return components.trainingService } diff --git a/training/service.go b/training/service.go index cb0862a0..877ef8ed 100644 --- a/training/service.go +++ b/training/service.go @@ -58,35 +58,12 @@ type DaemonService struct { storage *ModelStorage userStorage *ModelUserStorage pendingStorage *PendingModelStorage + publicStorage *PublicModelStorage serviceUrl string trainingMetadata *TrainingMetadata methodsMetadata map[string]*MethodMetadata } -func NewDaemonsService( - serviceMetaData *blockchain.ServiceMetadata, - organizationMetaData *blockchain.OrganizationMetaData, - channelService escrow.PaymentChannelService, - storage *ModelStorage, - userStorage *ModelUserStorage, - pendingStorage *PendingModelStorage, - serviceUrl string, - trainingMedadata *TrainingMetadata, - methodsMetadata map[string]*MethodMetadata, -) *DaemonService { - return &DaemonService{ - serviceMetaData: serviceMetaData, - organizationMetaData: organizationMetaData, - channelService: channelService, - storage: storage, - userStorage: userStorage, - pendingStorage: pendingStorage, - serviceUrl: serviceUrl, - trainingMetadata: trainingMedadata, - methodsMetadata: methodsMetadata, - } -} - func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest) (*ModelResponse, error) { zap.L().Debug("CreateModel request") @@ -154,22 +131,20 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest return modelResponse, err } -func (ds *DaemonService) buildPendingModelKey() *PendingModelKey { - return &PendingModelKey{ +func (ds *DaemonService) buildPublicModelKey() *PublicModelKey { + return &PublicModelKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), } } -func (ds *DaemonService) setPendingModelIds(modelIds *[]string) { - key := ds.buildPendingModelKey() - - data := &PendingModelData{ - ModelIDs: *modelIds, +func (ds *DaemonService) buildPendingModelKey() *PendingModelKey { + return &PendingModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), } - - ds.pendingStorage.Put(key, data) } func (ds *DaemonService) getPendingModelIds() (*PendingModelData, error) { @@ -225,7 +200,7 @@ func (ds *DaemonService) updateModelStatusworker(ctx context.Context, tasks <-ch } } -func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, interval time.Duration, orgID, serviceID, groupID string) { +func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, interval time.Duration) { ticker := time.NewTicker(interval) data, err := ds.getPendingModelIds() if err != nil { @@ -485,16 +460,18 @@ func (ds *DaemonService) getServiceClient() (conn *grpc.ClientConn, client Model return } -func (ds *DaemonService) createModelDetails(request *NewModelRequest, response *ModelID) (data *ModelData, err error) { +func (ds *DaemonService) createModelDetails(request *NewModelRequest, response *ModelID) (*ModelData, error) { key := ds.buildModelKey(response.ModelId) - data = ds.getModelDataToCreate(request, response) + data := ds.getModelDataToCreate(request, response) + //store the model details in etcd zap.L().Debug("createModelDetails", zap.Any("key", key)) - err = ds.storage.Put(key, data) + err := ds.storage.Put(key, data) if err != nil { zap.L().Error("can't put model in etcd", zap.Error(err)) - return + return nil, err } + // for every accessible address in the list, store the user address and all the model Ids associated with it for _, address := range data.AuthorizedAddresses { userKey := getModelUserKey(key, address) @@ -503,11 +480,21 @@ func (ds *DaemonService) createModelDetails(request *NewModelRequest, response * err = ds.userStorage.Put(userKey, userData) if err != nil { zap.L().Error("can't put in user storage", zap.Error(err)) - return + return nil, err } zap.L().Debug("creating training model", zap.String("userKey", userKey.String()), zap.String("userData", userData.String())) } - return + + if request.Model.IsPublic { + publicModelKey := ds.buildPublicModelKey() + err := ds.publicStorage.AddPublicModelId(publicModelKey, response.ModelId) + if err != nil { + zap.L().Error("error in adding model id to public model storage") + } + zap.L().Debug("adding model id to public model storage", zap.String("modelId", response.ModelId), zap.String("key", publicModelKey.String())) + } + + return data, nil } func getModelUserKey(key *ModelKey, address string) *ModelUserKey { @@ -648,7 +635,17 @@ func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data * // ensure only authorized use func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address string) (err error) { - key := &ModelUserKey{ + // check if model is public + publicModelKey := ds.buildPublicModelKey() + publicModels, ok, err := ds.publicStorage.Get(publicModelKey) + if ok && err == nil { + if slices.Contains(publicModels.ModelIDs, modelId) { + return + } + } + + // check user access if model is not public + userModelKey := &ModelUserKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), @@ -656,7 +653,7 @@ func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address //GRPCServiceName: serviceName, UserAddress: address, } - data, ok, err := ds.userStorage.Get(key) + data, ok, err := ds.userStorage.Get(userModelKey) if ok && err == nil { if !slices.Contains(data.ModelIds, modelId) { return fmt.Errorf("user %v, does not have access to model Id %v", address, modelId) @@ -720,12 +717,12 @@ func (ds *DaemonService) getModelData(modelID string) (data *ModelData, err erro return } -func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (response *ModelsResponse, err error) { +func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (*ModelsResponse, error) { if request == nil || request.Authorization == nil { return &ModelsResponse{}, fmt.Errorf("invalid request, no Authorization provided ") } - if err = ds.verifySignature(request.Authorization); err != nil { + if err := ds.verifySignature(request.Authorization); err != nil { return &ModelsResponse{}, fmt.Errorf("unable to access model: %v", err) } @@ -734,7 +731,7 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque // fmt.Errorf("Invalid request, no GrpcMethodName or GrpcServiceName provided") //} - key := &ModelUserKey{ + userModelKey := &ModelUserKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), @@ -744,7 +741,7 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque } modelDetailsArray := make([]*ModelResponse, 0) - if data, ok, err := ds.userStorage.Get(key); data != nil && ok && err == nil { + if data, ok, err := ds.userStorage.Get(userModelKey); data != nil && ok && err == nil { for _, modelId := range data.ModelIds { modelKey := &ModelKey{ OrganizationId: config.GetString(config.OrganizationId), @@ -761,14 +758,34 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque } } + publicModelKey := &PublicModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + } + + if data, ok, err := ds.publicStorage.Get(publicModelKey); data != nil && ok && err == nil { + for _, modelId := range data.ModelIDs { + modelKey := &ModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + //GRPCMethodName: request.GrpcMethodName, + //GRPCServiceName: request.GrpcServiceName, + ModelId: modelId, + } + if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { + boModel := convertModelDataToBO(modelData) + modelDetailsArray = append(modelDetailsArray, boModel) + } + } + } + for _, model := range modelDetailsArray { zap.L().Debug("Model", zap.String("Name", model.Name)) } - response = &ModelsResponse{ - ListOfModels: modelDetailsArray, - } - return + return &ModelsResponse{ListOfModels: modelDetailsArray}, nil } func (ds *DaemonService) getModelDataToCreate(request *NewModelRequest, response *ModelID) (data *ModelData) { @@ -903,7 +920,8 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re // NewTrainingService daemon self server func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData *blockchain.ServiceMetadata, - orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage, pendingStorage *PendingModelStorage) DaemonServer { + orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage, + pendingStorage *PendingModelStorage, publicStorage *PublicModelStorage) DaemonServer { linkerFiles := getFileDescriptors(serMetaData.ProtoFiles) serMetaData.ProtoDescriptors = linkerFiles @@ -923,12 +941,13 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData storage: storage, userStorage: userStorage, pendingStorage: pendingStorage, + publicStorage: publicStorage, serviceUrl: serviceURL, trainingMetadata: &trainMD, methodsMetadata: methodsMD, } - go daemonService.ManageUpdateModelStatusWorkers(context.Background(), 4*time.Second, "3", "2", "1") + go daemonService.ManageUpdateModelStatusWorkers(context.Background(), 4*time.Second) return daemonService } diff --git a/training/storage.go b/training/storage.go index ec323c2f..5bcb82d3 100644 --- a/training/storage.go +++ b/training/storage.go @@ -20,6 +20,10 @@ type PendingModelStorage struct { delegate storage.TypedAtomicStorage } +type PublicModelStorage struct { + delegate storage.TypedAtomicStorage +} + func NewUserModelStorage(atomicStorage storage.AtomicStorage) *ModelUserStorage { prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/userModelStorage") userModelStorage := storage.NewTypedAtomicStorageImpl( @@ -47,6 +51,15 @@ func NewPendingModelStorage(atomicStorage storage.AtomicStorage) *PendingModelSt return &PendingModelStorage{delegate: pendingModelStorage} } +func NewPublicModelStorage(atomicStorage storage.AtomicStorage) *PublicModelStorage { + prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/publicModelStorage") + publicModelStorage := storage.NewTypedAtomicStorageImpl( + prefixedStorage, serializePublicModelKey, reflect.TypeOf(PublicModelKey{}), utils.Serialize, utils.Deserialize, + reflect.TypeOf(PublicModelData{}), + ) + return &PublicModelStorage{delegate: publicModelStorage} +} + type ModelKey struct { OrganizationId string ServiceId string @@ -138,6 +151,24 @@ func (data *PendingModelData) String() string { return fmt.Sprintf("{DATA:%v}", data.ModelIDs) } +type PublicModelKey struct { + OrganizationId string + ServiceId string + GroupId string +} + +func (key *PublicModelKey) String() string { + return fmt.Sprintf("{ID:%v|%v|%v}", key.OrganizationId, key.ServiceId, key.GroupId) +} + +type PublicModelData struct { + ModelIDs []string +} + +func (data *PublicModelData) String() string { + return fmt.Sprintf("{DATA:%v}", data.ModelIDs) +} + func serializeModelKey(key any) (serialized string, err error) { modelKey := key.(*ModelKey) return modelKey.String(), nil @@ -235,6 +266,66 @@ func (storage *PendingModelStorage) Put(key *PendingModelKey, state *PendingMode return storage.delegate.Put(key, state) } +func (pendingStorage *PendingModelStorage) AddPendingModelId(key *PendingModelKey, modelId string) (err error) { + typedUpdateFunc := func(conditionValues []storage.TypedKeyValueData) (update []storage.TypedKeyValueData, ok bool, err error) { + if len(conditionValues) != 1 || conditionValues[0].Key != key { + return nil, false, fmt.Errorf("unexpected condition values or missing key") + } + + // Fetch the current list of pending model IDs from the storage + currentValue, ok, err := pendingStorage.delegate.Get(key) + if err != nil { + return nil, false, err + } + + var pendingModelData *PendingModelData + if currentValue == nil { + pendingModelData = &PendingModelData{ModelIDs: make([]string, 0, 100)} + } else { + pendingModelData = currentValue.(*PendingModelData) + } + + // Check if the modelId already exists + for _, currentModelId := range pendingModelData.ModelIDs { + if currentModelId == modelId { + // If the model ID already exists, no update is needed + return nil, false, nil + } + } + + // Add the new model ID to the list + pendingModelData.ModelIDs = append(pendingModelData.ModelIDs, modelId) + + // Prepare the updated values for the transaction + newValues := []storage.TypedKeyValueData{ + { + Key: key, + Value: pendingModelData, + Present: true, + }, + } + + return newValues, true, nil + } + + request := storage.TypedCASRequest{ + ConditionKeys: []any{key}, + RetryTillSuccessOrError: true, + Update: typedUpdateFunc, + } + + // Execute the transaction + ok, err := pendingStorage.delegate.ExecuteTransaction(request) + if err != nil { + return fmt.Errorf("transaction execution failed: %w", err) + } + if !ok { + return fmt.Errorf("transaction was not successful") + } + + return nil +} + func (storage *PendingModelStorage) PutIfAbsent(key *PendingModelKey, state *PendingModelData) (ok bool, err error) { return storage.delegate.PutIfAbsent(key, state) } @@ -243,3 +334,99 @@ func (storage *PendingModelStorage) CompareAndSwap(key *PendingModelKey, prevSta newState *PendingModelData) (ok bool, err error) { return storage.delegate.CompareAndSwap(key, prevState, newState) } + +func serializePublicModelKey(key any) (serialized string, err error) { + pendingModelKey := key.(*PublicModelKey) + return pendingModelKey.String(), nil +} + +func (storage *PublicModelStorage) Get(key *PublicModelKey) (state *PublicModelData, ok bool, err error) { + value, ok, err := storage.delegate.Get(key) + if err != nil || !ok { + return nil, ok, err + } + + return value.(*PublicModelData), ok, err +} + +func (storage *PublicModelStorage) GetAll() (states []*PublicModelData, err error) { + values, err := storage.delegate.GetAll() + if err != nil { + return + } + + return values.([]*PublicModelData), nil +} + +func (storage *PublicModelStorage) Put(key *PublicModelKey, state *PublicModelData) (err error) { + return storage.delegate.Put(key, state) +} + +func (publicStorage *PublicModelStorage) AddPublicModelId(key *PublicModelKey, modelId string) (err error) { + typedUpdateFunc := func(conditionValues []storage.TypedKeyValueData) (update []storage.TypedKeyValueData, ok bool, err error) { + if len(conditionValues) != 1 || conditionValues[0].Key != key { + return nil, false, fmt.Errorf("unexpected condition values or missing key") + } + + // Fetch the current list of public model IDs from the storage + currentValue, ok, err := publicStorage.delegate.Get(key) + if err != nil { + return nil, false, err + } + + var publicModelData *PublicModelData + if currentValue == nil { + publicModelData = &PublicModelData{ModelIDs: make([]string, 0, 100)} + } else { + publicModelData = currentValue.(*PublicModelData) + } + + // Check if the modelId already exists + for _, currentModelId := range publicModelData.ModelIDs { + if currentModelId == modelId { + // If the model ID already exists, no update is needed + return nil, false, nil + } + } + + // Add the new model ID to the list + publicModelData.ModelIDs = append(publicModelData.ModelIDs, modelId) + + // Prepare the updated values for the transaction + newValues := []storage.TypedKeyValueData{ + { + Key: key, + Value: publicModelData, + Present: true, + }, + } + + return newValues, true, nil + } + + request := storage.TypedCASRequest{ + ConditionKeys: []any{key}, + RetryTillSuccessOrError: true, + Update: typedUpdateFunc, + } + + // Execute the transaction + ok, err := publicStorage.delegate.ExecuteTransaction(request) + if err != nil { + return fmt.Errorf("transaction execution failed: %w", err) + } + if !ok { + return fmt.Errorf("transaction was not successful") + } + + return nil +} + +func (storage *PublicModelStorage) PutIfAbsent(key *PublicModelKey, state *PublicModelData) (ok bool, err error) { + return storage.delegate.PutIfAbsent(key, state) +} + +func (storage *PublicModelStorage) CompareAndSwap(key *PublicModelKey, prevState *PublicModelData, + newState *PublicModelData) (ok bool, err error) { + return storage.delegate.CompareAndSwap(key, prevState, newState) +} diff --git a/training/tests/integration/daemon_service_test.go b/training/tests/integration/daemon_service_test.go index 58d8bfc2..5d650a2d 100644 --- a/training/tests/integration/daemon_service_test.go +++ b/training/tests/integration/daemon_service_test.go @@ -22,7 +22,7 @@ type DaemonServiceSuite struct { modelStorage *training.ModelStorage userModelStorage *training.ModelUserStorage pendingModelStorage *training.PendingModelStorage - daemonService *training.DaemonService + daemonService training.DaemonServer modelKeys []*training.ModelKey grpcServer *grpc.Server } @@ -36,7 +36,7 @@ var ( func (suite *DaemonServiceSuite) SetupSuite() { suite.setupTestConfig() - modelKeys, modelStorage, userModelStorage, pendingModelStorage := suite.createTestModels() + modelKeys, modelStorage, userModelStorage, pendingModelStorage, publicModelStorage := suite.createTestModels() suite.modelKeys = modelKeys suite.modelStorage = modelStorage suite.userModelStorage = userModelStorage @@ -48,8 +48,14 @@ func (suite *DaemonServiceSuite) SetupSuite() { zap.L().Fatal("Error in initinalize organization metadata from json", zap.Error(err)) } - suite.daemonService = training.NewDaemonsService( - serviceMetadata, orgMetadata, nil, modelStorage, userModelStorage, pendingModelStorage, "http://localhost:5001", nil, nil, + suite.daemonService = training.NewTrainingService( + nil, + serviceMetadata, + orgMetadata, + modelStorage, + userModelStorage, + pendingModelStorage, + publicModelStorage, ) address := "localhost:5001" @@ -101,6 +107,7 @@ func (suite *DaemonServiceSuite) setupTestConfig() { }, "hooks": [] }, + "model_maintenance_endpoint": "http://localhost:5001", "payment_channel_storage_client": { "connection_timeout": "0s", "request_timeout": "0s", @@ -135,11 +142,13 @@ func (suite *DaemonServiceSuite) setupTestConfig() { config.SetVip(testConfig) } -func (suite *DaemonServiceSuite) createTestModels() ([]*training.ModelKey, *training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage) { +func (suite *DaemonServiceSuite) createTestModels() ([]*training.ModelKey, + *training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage, *training.PublicModelStorage) { memStorage := storage.NewMemStorage() modelStorage := training.NewModelStorage(memStorage) userModelStorage := training.NewUserModelStorage(memStorage) pendingModelStorage := training.NewPendingModelStorage(memStorage) + publicModelStorage := training.NewPublicModelStorage(memStorage) modelA := &training.ModelData{ IsPublic: true, @@ -209,7 +218,7 @@ func (suite *DaemonServiceSuite) createTestModels() ([]*training.ModelKey, *trai pendingModelStorage.Put(key, pendingModelsData) - return modelKeys, modelStorage, userModelStorage, pendingModelStorage + return modelKeys, modelStorage, userModelStorage, pendingModelStorage, publicModelStorage } func (suite *DaemonServiceSuite) TestRunDaemonService() { @@ -219,8 +228,6 @@ func (suite *DaemonServiceSuite) TestRunDaemonService() { ctx, cancel := context.WithDeadline(context.Background(), deadline) defer cancel() - suite.daemonService.ManageUpdateModelStatusWorkers(ctx, time.Second*3, "test_org_id", "service_id", "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=") - select { case <-ctx.Done(): zap.L().Info("Context done", zap.Error(ctx.Err())) diff --git a/training/tests/unit/storage_test.go b/training/tests/unit/storage_test.go index 9171ccc8..5b20a096 100644 --- a/training/tests/unit/storage_test.go +++ b/training/tests/unit/storage_test.go @@ -16,6 +16,7 @@ type ModelStorageSuite struct { storage *training.ModelStorage userStorage *training.ModelUserStorage pendingStorage *training.PendingModelStorage + publicStorage *training.PublicModelStorage organizationId string serviceId string groupId string @@ -38,6 +39,11 @@ func (suite *ModelStorageSuite) getPendingModelKey() *training.PendingModelKey { GroupId: suite.groupId} } +func (suite *ModelStorageSuite) getPublicModelKey() *training.PublicModelKey { + return &training.PublicModelKey{OrganizationId: suite.organizationId, ServiceId: suite.serviceId, + GroupId: suite.groupId} +} + func (suite *ModelStorageSuite) getModelData(modelId string) *training.ModelData { return &training.ModelData{ Status: training.Status_CREATED, @@ -68,11 +74,18 @@ func (suite *ModelStorageSuite) getPendingModelData(modelIds []string) *training } } +func (suite *ModelStorageSuite) getPublicModelData(modelIds []string) *training.PublicModelData { + return &training.PublicModelData{ + ModelIDs: modelIds, + } +} + func (suite *ModelStorageSuite) SetupSuite() { suite.memoryStorage = base_storage.NewMemStorage() suite.storage = training.NewModelStorage(suite.memoryStorage) - suite.userStorage = training.NewUserModelStorage(base_storage.NewMemStorage()) - suite.pendingStorage = training.NewPendingModelStorage(base_storage.NewMemStorage()) + suite.userStorage = training.NewUserModelStorage(suite.memoryStorage) + suite.pendingStorage = training.NewPendingModelStorage(suite.memoryStorage) + suite.publicStorage = training.NewPublicModelStorage(suite.memoryStorage) suite.accessibleAddress = make([]string, 2) suite.accessibleAddress[0] = "ADD1" suite.accessibleAddress[1] = "ADD2" @@ -86,7 +99,6 @@ func TestFreeCallUserStorageSuite(t *testing.T) { } func (suite *ModelStorageSuite) TestModelStorage_GetAll() { - key1 := suite.getModelKey("1") key2 := suite.getModelKey("2") data1 := suite.getModelData("1") @@ -150,6 +162,15 @@ func (suite *ModelStorageSuite) Test_serializePendingModelKey() { assert.Equal(suite.T(), expectedSerializedKey, serializedKey) } +func (suite *ModelStorageSuite) Test_serializePublicModelKey() { + expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId) + + key := suite.getPublicModelKey() + serializedKey := key.String() + + assert.Equal(suite.T(), expectedSerializedKey, serializedKey) +} + func (suite *ModelStorageSuite) TestModelStorage_CompareAndSwap() { key1 := suite.getModelKey("1") data1 := suite.getModelData("1") @@ -222,3 +243,58 @@ func (suite *ModelStorageSuite) TestPendingModelStorage_Get() { assert.NoError(suite.T(), err) assert.Equal(suite.T(), data, newData) } + +func (suite *ModelStorageSuite) TestPendingModelStorage_AddPendingModelId() { + key := suite.getPendingModelKey() + data := suite.getPendingModelData([]string{"1", "2"}) + + err := suite.pendingStorage.Put(key, data) + assert.NoError(suite.T(), err) + + newData, ok, err := suite.pendingStorage.Get(key) + assert.True(suite.T(), ok) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), data, newData) + + newModelId := "3" + data = suite.getPendingModelData([]string{"1", "2", "3"}) + err = suite.pendingStorage.AddPendingModelId(key, newModelId) + assert.NoError(suite.T(), err) + newData, ok, err = suite.pendingStorage.Get(key) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), data, newData) +} + +func (suite *ModelStorageSuite) TestPublicModelStorage_Get() { + key := suite.getPublicModelKey() + data := suite.getPublicModelData([]string{"1", "2"}) + + err := suite.publicStorage.Put(key, data) + assert.NoError(suite.T(), err) + + newData, ok, err := suite.publicStorage.Get(key) + assert.True(suite.T(), ok) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), data, newData) +} + +func (suite *ModelStorageSuite) TestPublicModelStorage_AddPublicModelId() { + key := suite.getPublicModelKey() + data := suite.getPublicModelData([]string{"1", "2"}) + + err := suite.publicStorage.Put(key, data) + assert.NoError(suite.T(), err) + + newData, ok, err := suite.publicStorage.Get(key) + assert.True(suite.T(), ok) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), data, newData) + + newModelId := "3" + data = suite.getPublicModelData([]string{"1", "2", "3"}) + err = suite.publicStorage.AddPublicModelId(key, newModelId) + assert.NoError(suite.T(), err) + newData, ok, err = suite.publicStorage.Get(key) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), data, newData) +} From aa2db13e5e1e7ef316e25497334dfd02e86e94ff Mon Sep 17 00:00:00 2001 From: Elmir Iskhakov <86847688+elmiringos@users.noreply.github.com> Date: Fri, 17 Jan 2025 18:20:11 +0300 Subject: [PATCH 11/22] Updated tests and owner verification method (#13) * feat(training): add error.go for training module * feat(training/service): add custom errors, update error handling logic in verifySignerHasAccessToTheModel, add verifyCreatedByAddress * test(training): add new functional tests for GetModel, GetAllModels, CreateModel, ManageUpdateStatusWorkers * refactor(training): rename daemon_service_test.go to service_test.go --- snetd/cmd/channel.go | 2 +- snetd/cmd/components.go | 2 +- snetd/cmd/free_call_users.go | 3 +- training/error.go | 40 ++ training/mock.go | 129 ++-- training/service.go | 181 ++--- training/service_test.go | 502 -------------- training/tests/functional/service_test.go | 645 ++++++++++++++++++ .../test_provider_service.go | 0 .../tests/integration/daemon_service_test.go | 244 ------- training/util.go | 3 +- 11 files changed, 835 insertions(+), 916 deletions(-) create mode 100644 training/error.go delete mode 100644 training/service_test.go create mode 100644 training/tests/functional/service_test.go rename training/tests/{integration => functional}/test_provider_service.go (100%) delete mode 100644 training/tests/integration/daemon_service_test.go diff --git a/snetd/cmd/channel.go b/snetd/cmd/channel.go index b80e54b1..bb1ce4ea 100644 --- a/snetd/cmd/channel.go +++ b/snetd/cmd/channel.go @@ -39,7 +39,7 @@ func newChannelCommand(cmd *cobra.Command, args []string, components *Components return } -func getPaymentChannelId(cmd *cobra.Command) (id *big.Int, err error) { +func getPaymentChannelId(*cobra.Command) (id *big.Int, err error) { if paymentChannelId == "" { return nil, nil } diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index fb4920d1..d411167e 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -607,7 +607,7 @@ func (components *Components) TrainingService() training.DaemonServer { return components.trainingService } if !config.GetBool(config.BlockchainEnabledKey) { - components.trainingService = training.NoTrainingService{} + components.trainingService = training.NoTrainingDaemonServer{} return components.trainingService } diff --git a/snetd/cmd/free_call_users.go b/snetd/cmd/free_call_users.go index d788ba26..6f2123e8 100644 --- a/snetd/cmd/free_call_users.go +++ b/snetd/cmd/free_call_users.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/singnet/snet-daemon/v5/blockchain" "github.com/singnet/snet-daemon/v5/config" "github.com/singnet/snet-daemon/v5/escrow" @@ -71,7 +72,7 @@ func newFreeCallResetCountCommand(cmd *cobra.Command, args []string, pComponents return } -func getUserId(cmd *cobra.Command) (userId string, err error) { +func getUserId(*cobra.Command) (userId string, err error) { return freeCallUserId, nil } diff --git a/training/error.go b/training/error.go new file mode 100644 index 00000000..6ea8badf --- /dev/null +++ b/training/error.go @@ -0,0 +1,40 @@ +package training + +import ( + "errors" + "fmt" +) + +// Base Error +var ( + ErrInvalidRequest = errors.New("invalud request") + ErrUpdatingModel = errors.New("error in updating model state") + ErrServiceInvocation = errors.New("error in invoking service for model training") + ErrServiceIssue = errors.New("issue with service") + ErrAccessToModel = errors.New("unable to access model") + ErrDaemonStorage = errors.New("daemon storage error") + ErrModelDoesntExist = errors.New("model doesn't exist") +) + +// Specific Error +var ( + ErrEmptyResponse = fmt.Errorf("%w: service returned empty response", ErrServiceInvocation) + ErrEmptyModelIDFromService = fmt.Errorf("%w: service returned empty modelID", ErrServiceInvocation) + ErrServiceIssueValidateModel = fmt.Errorf("%w: ValidateModel method error", ErrServiceIssue) + ErrNoAuthorization = fmt.Errorf("%w: no authorization provided", ErrInvalidRequest) + ErrBadAuthorization = fmt.Errorf("%w: bad authorization provided", ErrInvalidRequest) + ErrNoGRPCServiceOrMethod = fmt.Errorf("%w: no grpc_service_name or grpc_method_name provided", ErrInvalidRequest) + ErrGetUserModelStorage = fmt.Errorf("%w: error in getting data from user model storage", ErrDaemonStorage) + ErrGetModelStorage = fmt.Errorf("%w: error in getting data from model storage", ErrDaemonStorage) + ErrPutModelStorage = fmt.Errorf("%w: error in putting data to model storage", ErrDaemonStorage) + ErrEmptyModelID = fmt.Errorf("%w: model id can't be empty", ErrInvalidRequest) + ErrNotOwnerModel = fmt.Errorf("%w: only owner can change the model state", ErrUpdatingModel) +) + +// WrapError formats and wraps an error with additional context. +func WrapError(baseErr error, message string) error { + if baseErr == nil { + return nil + } + return fmt.Errorf("%w: %s", baseErr, message) +} diff --git a/training/mock.go b/training/mock.go index 7b05dc3a..30288d9b 100644 --- a/training/mock.go +++ b/training/mock.go @@ -3,136 +3,113 @@ package training import ( "context" "fmt" + "google.golang.org/protobuf/types/known/emptypb" ) -type NoModelSupportService struct { -} - -type NoTrainingService struct { -} - -func (n NoTrainingService) mustEmbedUnimplementedDaemonServer() { - panic("implement me") -} - -func (n NoTrainingService) CreateModel(ctx context.Context, request *NewModelRequest) (*ModelResponse, error) { - panic("implement me") -} - -func (n NoTrainingService) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { - panic("implement me") -} - -func (n NoTrainingService) UploadAndValidate(server Daemon_UploadAndValidateServer) error { - panic("implement me") -} - -func (n NoTrainingService) ValidateModel(ctx context.Context, request *AuthValidateRequest) (*StatusResponse, error) { - panic("implement me") -} - -func (n NoTrainingService) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { - panic("implement me") -} - -func (n NoTrainingService) TrainModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { - panic("implement me") +type NoModelSupportTrainingService struct { } -func (n NoTrainingService) DeleteModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { - panic("implement me") +type NoTrainingDaemonServer struct { } -func (n NoTrainingService) GetTrainingMetadata(ctx context.Context, empty *emptypb.Empty) (*TrainingMetadata, error) { +func (n NoTrainingDaemonServer) mustEmbedUnimplementedDaemonServer() { panic("implement me") } -func (n NoTrainingService) GetAllModels(ctx context.Context, request *AllModelsRequest) (*ModelsResponse, error) { - panic("implement me") +func (n NoTrainingDaemonServer) CreateModel(ctx context.Context, request *NewModelRequest) (*ModelResponse, error) { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (n NoTrainingService) GetModel(ctx context.Context, request *CommonRequest) (*ModelResponse, error) { - panic("implement me") +func (n NoTrainingDaemonServer) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (n NoTrainingService) UpdateModel(ctx context.Context, request *UpdateModelRequest) (*ModelResponse, error) { - panic("implement me") +func (n NoTrainingDaemonServer) UploadAndValidate(server Daemon_UploadAndValidateServer) error { + return fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (n NoTrainingService) GetMethodMetadata(ctx context.Context, request *MethodMetadataRequest) (*MethodMetadata, error) { - panic("implement me") +func (n NoTrainingDaemonServer) ValidateModel(ctx context.Context, request *AuthValidateRequest) (*StatusResponse, error) { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) CreateModel(ctx context.Context, model *NewModel) (*ModelID, error) { - panic("implement me") -} - -func (service ModelService) ValidateModelPrice(ctx context.Context, request *ValidateRequest) (*PriceInBaseUnit, error) { - panic("implement me") +func (n NoTrainingDaemonServer) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { + return nil, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) UploadAndValidate(server Model_UploadAndValidateServer) error { - panic("implement me") +func (n NoTrainingDaemonServer) TrainModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) ValidateModel(ctx context.Context, request *ValidateRequest) (*StatusResponse, error) { - panic("implement me") +func (n NoTrainingDaemonServer) DeleteModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) TrainModelPrice(ctx context.Context, id *ModelID) (*PriceInBaseUnit, error) { - panic("implement me") +func (n NoTrainingDaemonServer) GetTrainingMetadata(ctx context.Context, empty *emptypb.Empty) (*TrainingMetadata, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) TrainModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { - panic("implement me") +func (n NoTrainingDaemonServer) GetAllModels(ctx context.Context, request *AllModelsRequest) (*ModelsResponse, error) { + return &ModelsResponse{ListOfModels: []*ModelResponse{}}, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) DeleteModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { - panic("implement me") +func (n NoTrainingDaemonServer) GetModel(ctx context.Context, request *CommonRequest) (*ModelResponse, error) { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) GetModelStatus(ctx context.Context, id *ModelID) (*StatusResponse, error) { - panic("implement me") +func (n NoTrainingDaemonServer) UpdateModel(ctx context.Context, request *UpdateModelRequest) (*ModelResponse, error) { + return &ModelResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (service ModelService) mustEmbedUnimplementedModelServer() { - panic("implement me") +func (n NoTrainingDaemonServer) GetMethodMetadata(ctx context.Context, request *MethodMetadataRequest) (*MethodMetadata, error) { + return nil, fmt.Errorf("service end point is not defined or is invalid , please contact the AI developer") } -func (n NoModelSupportService) CreateModel(ctx context.Context, model *NewModel) (*ModelID, error) { +func (n NoModelSupportTrainingService) CreateModel(ctx context.Context, model *NewModel) (*ModelID, error) { return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) ValidateModelPrice(ctx context.Context, request *ValidateRequest) (*PriceInBaseUnit, error) { +func (n NoModelSupportTrainingService) ValidateModelPrice(ctx context.Context, request *ValidateRequest) (*PriceInBaseUnit, error) { return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) UploadAndValidate(server Model_UploadAndValidateServer) error { +func (n NoModelSupportTrainingService) UploadAndValidate(server Model_UploadAndValidateServer) error { return fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) ValidateModel(ctx context.Context, request *ValidateRequest) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") +func (n NoModelSupportTrainingService) ValidateModel(ctx context.Context, request *ValidateRequest) (*StatusResponse, error) { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) TrainModelPrice(ctx context.Context, id *ModelID) (*PriceInBaseUnit, error) { +func (n NoModelSupportTrainingService) TrainModelPrice(ctx context.Context, id *ModelID) (*PriceInBaseUnit, error) { return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) TrainModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") +func (n NoModelSupportTrainingService) TrainModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) DeleteModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") +func (n NoModelSupportTrainingService) DeleteModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) GetModelStatus(ctx context.Context, id *ModelID) (*StatusResponse, error) { - return nil, fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") +func (n NoModelSupportTrainingService) GetModelStatus(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return &StatusResponse{Status: Status_ERRORED}, + fmt.Errorf("service end point is not defined or is invalid, please contact the AI developer") } -func (n NoModelSupportService) mustEmbedUnimplementedModelServer() { +func (n NoModelSupportTrainingService) mustEmbedUnimplementedModelServer() { panic("implement me") } diff --git a/training/service.go b/training/service.go index 877ef8ed..1f81d64c 100644 --- a/training/service.go +++ b/training/service.go @@ -69,20 +69,17 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest zap.L().Debug("CreateModel request") if request == nil || request.Authorization == nil { - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no authorization provided") + return &ModelResponse{Status: Status_ERRORED}, ErrNoAuthorization } if err := ds.verifySignature(request.Authorization); err != nil { zap.L().Error("unable to create model, bad authorization provided", zap.Error(err)) - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("bad authorization provided: %v", err) + return &ModelResponse{Status: Status_ERRORED}, ErrBadAuthorization } if request.GetModel().GrpcServiceName == "" || request.GetModel().GrpcMethodName == "" { zap.L().Error("invalid request, no grpc_method_name or grpc_service_name provided") - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no grpc_service_name or grpc_method_name provided") + return &ModelResponse{Status: Status_ERRORED}, ErrNoGRPCServiceOrMethod } request.Model.ServiceId = config.GetString(config.ServiceId) @@ -95,28 +92,24 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest conn, client, err := ds.getServiceClient() if err != nil { zap.L().Error("[CreateModel] unable to getServiceClient", zap.Error(err)) - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("error in invoking service for Model Training %v", err) + return &ModelResponse{Status: Status_ERRORED}, WrapError(ErrServiceInvocation, err.Error()) } responseModelID, errClient := client.CreateModel(c, request.Model) closeConn(conn) if errClient != nil { zap.L().Error("[CreateModel] unable to call CreateModel", zap.Error(errClient)) - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("error in invoking service for Model Training %v", errClient) + return &ModelResponse{Status: Status_ERRORED}, WrapError(ErrServiceInvocation, errClient.Error()) } if responseModelID == nil { zap.L().Error("[CreateModel] CreateModel returned null response") - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("error in invoking service for model training: service return empty response") + return &ModelResponse{Status: Status_ERRORED}, ErrEmptyResponse } if responseModelID.ModelId == "" { zap.L().Error("[CreateModel] CreateModel returned empty modelID") - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("error in invoking service for model training: service return empty modelID") + return &ModelResponse{Status: Status_ERRORED}, ErrEmptyModelID } //store the details in etcd @@ -125,7 +118,7 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest data, err := ds.createModelDetails(request, responseModelID) if err != nil { zap.L().Error("[CreateModel] Can't save model", zap.Error(err)) - return nil, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) + return nil, WrapError(ErrDaemonStorage, err.Error()) } modelResponse := BuildModelResponse(data, Status_CREATED) return modelResponse, err @@ -240,7 +233,7 @@ func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthVa if client == nil || err != nil { return &PriceInBaseUnit{ Price: 0, - }, fmt.Errorf("issue with service: %v", err) + }, WrapError(ErrServiceIssue, err.Error()) } price, err := client.ValidateModelPrice(context.Background(), &ValidateRequest{ ModelId: request.ModelId, @@ -249,7 +242,7 @@ func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthVa closeConn(conn) if err != nil || price == nil { zap.L().Error("issue with ValidateModelPrice", zap.Error(err)) - return nil, fmt.Errorf("issue with service: %v", err) + return nil, WrapError(ErrServiceIssue, err.Error()) } return price, nil } @@ -309,7 +302,7 @@ func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidat if client == nil || err != nil { return &StatusResponse{ Status: Status_ERRORED, - }, fmt.Errorf("issue with service: %v", err) + }, WrapError(ErrServiceIssue, err.Error()) } price, err := client.ValidateModel(ctx, &ValidateRequest{ ModelId: request.ModelId, @@ -317,7 +310,7 @@ func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidat }) closeConn(conn) if err != nil { - return nil, fmt.Errorf("[ValidateModel] issue with service: %v", err) + return nil, WrapError(ErrServiceIssue, err.Error()) } return price, nil } @@ -327,14 +320,14 @@ func (ds *DaemonService) TrainModelPrice(ctx context.Context, request *CommonReq if client == nil || err != nil { return &PriceInBaseUnit{ Price: 0, - }, fmt.Errorf("issue with service: %v", err) + }, WrapError(ErrServiceIssue, err.Error()) } price, err := client.TrainModelPrice(ctx, &ModelID{ ModelId: request.ModelId, }) closeConn(conn) if err != nil { - return nil, fmt.Errorf("issue with service: %v", err) + return nil, WrapError(ErrServiceIssue, err.Error()) } return price, nil } @@ -345,7 +338,7 @@ func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) zap.L().Error("issue with service", zap.Error(err)) return &StatusResponse{ Status: Status_ERRORED, - }, fmt.Errorf("issue with service: %v", err) + }, WrapError(ErrServiceIssue, err.Error()) } statusResp, err := client.TrainModel(ctx, &ModelID{ ModelId: request.ModelId, @@ -355,7 +348,7 @@ func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) zap.L().Error("[TrainModel] issue with service", zap.Error(err)) return &StatusResponse{ Status: Status_ERRORED, - }, fmt.Errorf("issue with service: %v", err) + }, WrapError(ErrServiceIssue, err.Error()) } return statusResp, nil } @@ -367,26 +360,28 @@ func (ds *DaemonService) GetTrainingMetadata(ctx context.Context, empty *emptypb func (ds *DaemonService) UpdateModel(ctx context.Context, request *UpdateModelRequest) (*ModelResponse, error) { if request == nil || request.Authorization == nil { - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no authorization provided") + return &ModelResponse{Status: Status_ERRORED}, ErrNoAuthorization } if err := ds.verifySignature(request.Authorization); err != nil { return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("unable to access model: %v", err) + WrapError(ErrAccessToModel, err.Error()) } if err := ds.verifySignerHasAccessToTheModel(request.ModelId, request.Authorization.SignerAddress); err != nil { return &ModelResponse{}, - fmt.Errorf("unable to access model: %v", err) + WrapError(ErrAccessToModel, err.Error()) } if request.ModelId == "" { - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no model id provided") + return &ModelResponse{Status: Status_ERRORED}, ErrEmptyModelID + } + if err := ds.verifyCreatedByAddress(request.ModelId, request.Authorization.SignerAddress); err != nil { + return &ModelResponse{}, err } zap.L().Info("Updating model") data, err := ds.updateModelDetails(request) if err != nil || data == nil { - return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) + return &ModelResponse{Status: Status_ERRORED}, + WrapError(ErrDaemonStorage, err.Error()) } return BuildModelResponse(data, data.Status), nil } @@ -396,7 +391,7 @@ func (ds *DaemonService) GetMethodMetadata(ctx context.Context, request *MethodM data, err := ds.getModelData(request.ModelId) if err != nil { zap.L().Error("[GetMethodMetadata] can't get model data", zap.Error(err)) - return nil, fmt.Errorf(" can't get model data: %v", err) + return nil, WrapError(ErrGetModelStorage, err.Error()) } request.GrpcMethodName = data.GRPCMethodName request.GrpcServiceName = data.GRPCServiceName @@ -502,9 +497,7 @@ func getModelUserKey(key *ModelKey, address string) *ModelUserKey { OrganizationId: key.OrganizationId, ServiceId: key.ServiceId, GroupId: key.GroupId, - //GRPCMethodName: key.GRPCMethodName, - //GRPCServiceName: key.GRPCServiceName, - UserAddress: address, + UserAddress: address, } } @@ -525,10 +518,8 @@ func (ds *DaemonService) getModelUserData(key *ModelKey, address string) *ModelU OrganizationId: key.OrganizationId, ServiceId: key.ServiceId, GroupId: key.GroupId, - //GRPCMethodName: key.GRPCMethodName, - //GRPCServiceName: key.GRPCServiceName, - UserAddress: address, - ModelIds: modelIds, + UserAddress: address, + ModelIds: modelIds, } } @@ -628,7 +619,6 @@ func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data * zap.L().Error("Error in putting data in storage", zap.Error(err)) } } - } return } @@ -649,16 +639,47 @@ func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: methodName, - //GRPCServiceName: serviceName, - UserAddress: address, + UserAddress: address, } - data, ok, err := ds.userStorage.Get(userModelKey) - if ok && err == nil { - if !slices.Contains(data.ModelIds, modelId) { - return fmt.Errorf("user %v, does not have access to model Id %v", address, modelId) - } + userModelsData, ok, err := ds.userStorage.Get(userModelKey) + + if err != nil { + return WrapError(ErrGetUserModelStorage, err.Error()) + } + + if !ok { + return fmt.Errorf("user %v, does not have access to model Id %v", address, modelId) + } + + if !slices.Contains(userModelsData.ModelIds, modelId) { + return fmt.Errorf("user %v, does not have access to model Id %v", address, modelId) + } + + return +} + +// ensure only owner can update the model state +func (ds *DaemonService) verifyCreatedByAddress(modelId, address string) (err error) { + modelKey := &ModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + ModelId: modelId, + } + + modelData, ok, err := ds.storage.Get(modelKey) + if err != nil { + return WrapError(ErrGetModelStorage, err.Error()) + } + + if !ok { + return WrapError(ErrGetModelStorage, fmt.Sprintf("model data doesn't for key: %s", modelKey)) + } + + if modelData.CreatedByAddress != address { + return ErrNotOwnerModel } + return } @@ -674,12 +695,12 @@ func (ds *DaemonService) updateModelStatus(request *CommonRequest, newStatus Sta data, ok, err = ds.storage.Get(key) if err != nil || !ok || data == nil { zap.L().Error("[updateModelStatus] can't get model data from etcd", zap.Error(err)) - return data, errors.New("can't get model data from etcd") + return data, WrapError(ErrGetModelStorage, err.Error()) } data.Status = newStatus if err = ds.storage.Put(key, data); err != nil { zap.L().Error("[updateModelStatus] issue with retrieving model data from storage", zap.Error(err)) - return data, fmt.Errorf("can't get model data from etcd: %s", err) + return data, WrapError(ErrGetModelStorage, err.Error()) } return } @@ -689,9 +710,7 @@ func (ds *DaemonService) buildModelKey(modelID string) (key *ModelKey) { OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.Model.GrpcMethodName, - //GRPCServiceName: request.Model.GrpcServiceName, - ModelId: modelID, + ModelId: modelID, } return } @@ -701,9 +720,7 @@ func (ds *DaemonService) getModelKey(modelID string) (key *ModelKey) { OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.UpdateModelDetails.GrpcMethodName, - //GRPCServiceName: request.UpdateModelDetails.GrpcServiceName, - ModelId: modelID, + ModelId: modelID, } return } @@ -719,25 +736,17 @@ func (ds *DaemonService) getModelData(modelID string) (data *ModelData, err erro func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (*ModelsResponse, error) { if request == nil || request.Authorization == nil { - return &ModelsResponse{}, - fmt.Errorf("invalid request, no Authorization provided ") + return &ModelsResponse{}, ErrNoAuthorization } if err := ds.verifySignature(request.Authorization); err != nil { - return &ModelsResponse{}, - fmt.Errorf("unable to access model: %v", err) + return &ModelsResponse{}, ErrBadAuthorization } - //if request.GetGrpcMethodName() == "" || request.GetGrpcServiceName() == "" { - // return &AccessibleModelsResponse{}, - // fmt.Errorf("Invalid request, no GrpcMethodName or GrpcServiceName provided") - //} userModelKey := &ModelUserKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.GrpcMethodName, - //GRPCServiceName: request.GrpcServiceName, - UserAddress: request.Authorization.SignerAddress, + UserAddress: request.Authorization.SignerAddress, } modelDetailsArray := make([]*ModelResponse, 0) @@ -747,9 +756,7 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.GrpcMethodName, - //GRPCServiceName: request.GrpcServiceName, - ModelId: modelId, + ModelId: modelId, } if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { boModel := convertModelDataToBO(modelData) @@ -770,9 +777,7 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.GrpcMethodName, - //GRPCServiceName: request.GrpcServiceName, - ModelId: modelId, + ModelId: modelId, } if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { boModel := convertModelDataToBO(modelData) @@ -833,30 +838,29 @@ func BuildModelResponse(data *ModelData, status Status) *ModelResponse { func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*StatusResponse, error) { if req == nil || req.Authorization == nil { - return &StatusResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no Authorization provided") + return &StatusResponse{Status: Status_ERRORED}, ErrNoAuthorization } if req.ModelId == "" { - return &StatusResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request: ModelId is empty") + return &StatusResponse{Status: Status_ERRORED}, ErrEmptyModelID } if err := ds.verifySignature(req.Authorization); err != nil { return &StatusResponse{Status: Status_ERRORED}, - fmt.Errorf("unable to access model , %v", err) + WrapError(ErrAccessToModel, err.Error()) } if err := ds.verifySignerHasAccessToTheModel(req.ModelId, req.Authorization.SignerAddress); err != nil { return &StatusResponse{}, - fmt.Errorf("unable to access model , %v", err) + WrapError(ErrAccessToModel, err.Error()) } ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) defer cancel() conn, client, err := ds.getServiceClient() if err != nil { - return &StatusResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for Model Training") + return &StatusResponse{Status: Status_ERRORED}, + WrapError(ErrServiceInvocation, err.Error()) } response, errModel := client.DeleteModel(ctx, &ModelID{ModelId: req.ModelId}) closeConn(conn) @@ -867,7 +871,7 @@ func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*St data, err := ds.deleteModelDetails(req) if err != nil || data == nil { zap.L().Error("issue with deleting ModelId in storage", zap.Error(err)) - return response, fmt.Errorf("issue with deleting Model Id in Storage %v", err) + return response, WrapError(ErrDaemonStorage, fmt.Sprintf("issue with deleting Model %v", err)) } //responseData := BuildModelResponse(data, response.Status) return &StatusResponse{Status: Status_DELETED}, err @@ -876,20 +880,17 @@ func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*St func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (response *ModelResponse, err error) { if request == nil || request.Authorization == nil { - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request, no Authorization provided , %v", err) + return &ModelResponse{Status: Status_ERRORED}, ErrNoAuthorization } if err = ds.verifySignature(request.Authorization); err != nil { - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("unable to access model , %v", err) + return &ModelResponse{Status: Status_ERRORED}, ErrBadAuthorization + } + if request.ModelId == "" { + return &ModelResponse{Status: Status_ERRORED}, ErrEmptyModelID } if err = ds.verifySignerHasAccessToTheModel(request.ModelId, request.Authorization.SignerAddress); err != nil { return &ModelResponse{}, - fmt.Errorf("unable to access model , %v", err) - } - if request.ModelId == "" { - return &ModelResponse{Status: Status_ERRORED}, - fmt.Errorf("invalid request: ModelId can't be empty") + WrapError(ErrAccessToModel, err.Error()) } ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) @@ -952,7 +953,7 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData return daemonService } - return &NoTrainingService{} + return &NoTrainingDaemonServer{} } // parseTrainingMetadata TODO add comment diff --git a/training/service_test.go b/training/service_test.go deleted file mode 100644 index a2727883..00000000 --- a/training/service_test.go +++ /dev/null @@ -1,502 +0,0 @@ -package training - -import ( - "bytes" - "crypto/ecdsa" - "fmt" - "math/big" - "net" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/crypto" - "github.com/singnet/snet-daemon/v5/blockchain" - "github.com/singnet/snet-daemon/v5/config" - "github.com/singnet/snet-daemon/v5/storage" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "go.uber.org/zap" - "golang.org/x/net/context" - "google.golang.org/grpc" -) - -var count int = 0 - -type ModelServiceTestSuite struct { - suite.Suite - serviceURL string - server *grpc.Server - mockService MockServiceModelGRPCImpl - service ModelServer - serviceNotImplemented ModelServer - senderPvtKy *ecdsa.PrivateKey - senderAddress common.Address - - alternateUserPvtKy *ecdsa.PrivateKey - alternateUserAddress common.Address -} - -func TestModelServiceTestSuite(t *testing.T) { - suite.Run(t, new(ModelServiceTestSuite)) -} - -func (suite *ModelServiceTestSuite) getGRPCServerAndServe() { - config.Vip().Set(config.ModelMaintenanceEndPoint, "http://localhost:2222") - config.Vip().Set(config.ModelTrainingEndpoint, "http://localhost:2222") - ch := make(chan int) - go func() { - listener, err := net.Listen("tcp", ":2222") - if err != nil { - panic(err) - } - suite.server = grpc.NewServer() - - RegisterModelServer(suite.server, suite.mockService) - ch <- 0 - suite.server.Serve(listener) - - }() - - _ = <-ch -} - -func (suite *ModelServiceTestSuite) SetupSuite() { - suite.serviceNotImplemented = NewModelService(nil, nil, nil, nil, nil) - config.Vip().Set(config.ModelMaintenanceEndPoint, "localhost:2222") - suite.mockService = MockServiceModelGRPCImpl{} - suite.serviceURL = config.GetString(config.ModelMaintenanceEndPoint) - suite.getGRPCServerAndServe() - - testJsonOrgGroupData := "{ \"org_name\": \"organization_name\", \"org_id\": \"ExampleOrganizationId\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"group_id\": \"88ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" - testJsonData := "{ \"version\": 1, \"display_name\": \"Example1\", \"encoding\": \"grpc\", \"service_type\": \"grpc\", \"payment_expiration_threshold\": 40320, \"model_ipfs_hash\": \"Qmdiq8Hu6dYiwp712GtnbBxagyfYyvUY1HYqkH7iN76UCc\", " + - " \"mpe_address\": \"0x7E6366Fbe3bdfCE3C906667911FC5237Cc96BD08\", \"groups\": [ { \"free_calls\": 12, \"free_call_signer_address\": \"0x94d04332C4f5273feF69c4a52D24f42a3aF1F207\", \"endpoints\": [\"http://34.344.33.1:2379\",\"http://34.344.33.1:2389\"], \"group_id\": \"88ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\",\"group_name\": \"default_group\", \"pricing\": [ { \"price_model\": \"fixed_price\", \"price_in_cogs\": 2 }, { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"default\":true, \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] }, { \"endpoints\": [\"http://97.344.33.1:2379\",\"http://67.344.33.1:2389\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"pricing\": [ { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] } ] } " - - orgMetaData, _ := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) - serviceMetaData, _ := blockchain.InitServiceMetaDataFromJson([]byte(testJsonData)) - suite.service = NewTrainingService(nil, serviceMetaData, orgMetaData, - NewModelStorage(storage.NewMemStorage()), NewUerModelStorage(storage.NewMemStorage())) - suite.senderPvtKy, _ = crypto.GenerateKey() - suite.senderAddress = crypto.PubkeyToAddress(suite.senderPvtKy.PublicKey) - suite.alternateUserPvtKy, _ = crypto.GenerateKey() - suite.alternateUserAddress = crypto.PubkeyToAddress(suite.alternateUserPvtKy.PublicKey) - - config.Vip().Set(config.ModelMaintenanceEndPoint, "localhost:2222") - -} - -type MockServiceModelGRPCImpl struct { - count int -} - -func (m MockServiceModelGRPCImpl) mustEmbedUnimplementedModelServer() { - //TODO implement me - panic("implement me") -} - -func (m MockServiceModelGRPCImpl) CreateModel(context context.Context, request *CreateModelRequest) (*ModelDetailsResponse, error) { - zap.L().Info("In Service CreateModel") - count = count + 1 - zap.L().Debug("Count", zap.Int("value", count)) - return &ModelDetailsResponse{Status: Status_CREATED, - ModelDetails: &ModelDetails{ - ModelId: fmt.Sprintf("%d", count), - }}, nil -} - -func (m MockServiceModelGRPCImpl) UpdateModelAccess(context context.Context, request *UpdateModelRequest) (*ModelDetailsResponse, error) { - return &ModelDetailsResponse{Status: Status_IN_PROGRESS, - ModelDetails: &ModelDetails{ - ModelId: request.UpdateModelDetails.ModelId, - }}, nil -} - -func (m MockServiceModelGRPCImpl) DeleteModel(context context.Context, request *UpdateModelRequest) (*ModelDetailsResponse, error) { - return &ModelDetailsResponse{Status: Status_DELETED, - ModelDetails: &ModelDetails{ - ModelId: request.UpdateModelDetails.ModelId, - }}, nil -} - -func (m MockServiceModelGRPCImpl) GetModelStatus(context context.Context, request *ModelDetailsRequest) (*ModelDetailsResponse, error) { - return &ModelDetailsResponse{Status: Status_IN_PROGRESS, - ModelDetails: &ModelDetails{ - ModelId: request.ModelDetails.ModelId, - }}, nil -} - -//func (m MockServiceModelGRPCImpl) GetAllModels(context context.Context, request *AccessibleModelsRequest) (*AccessibleModelsResponse, error) { -// //Ideally client should take a list of all models and update the status of each and send back a response -// return &AccessibleModelsResponse{}, nil -//} - -func (suite *ModelServiceTestSuite) TearDownSuite() { - suite.server.GracefulStop() -} - -func (suite *ModelServiceTestSuite) getSignature(text string, blockNumber int, privateKey *ecdsa.PrivateKey) (signature []byte) { - message := bytes.Join([][]byte{ - []byte(text), - crypto.PubkeyToAddress(privateKey.PublicKey).Bytes(), - math.U256Bytes(big.NewInt(int64(blockNumber))), - }, nil) - hash := crypto.Keccak256( - blockchain.HashPrefix32Bytes, - crypto.Keccak256(message), - ) - signature, err := crypto.Sign(hash, suite.senderPvtKy) - if err != nil { - panic(fmt.Sprintf("Cannot sign test message: %v", err)) - } - return signature -} - -func (suite *ModelServiceTestSuite) TestModelService_UndefinedTrainingService() { - //when AI developer has not implemented the training.prot , ensure we get back an error when daemon is called - response, err := suite.serviceNotImplemented.CreateModel(context.TODO(), &CreateModelRequest{}) - assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), response.Status, Status_ERRORED) - - response2, err := suite.serviceNotImplemented.UpdateModelAccess(context.TODO(), &UpdateModelRequest{}) - assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), response2.Status, Status_ERRORED) - - response3, err := suite.serviceNotImplemented.DeleteModel(context.TODO(), &UpdateModelRequest{}) - assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), response3.Status, Status_ERRORED) - - response4, err := suite.serviceNotImplemented.GetModelStatus(context.TODO(), &ModelDetailsRequest{}) - assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), response4.Status, Status_ERRORED) - - response5, err := suite.serviceNotImplemented.GetAllModels(context.TODO(), &AccessibleModelsRequest{}) - assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), len(response5.ListOfModels), 0) - -} - -func (suite *ModelServiceTestSuite) TestModelService_CreateModel() { - //No Authorization request - response, err := suite.service.CreateModel(context.TODO(), nil) - assert.NotNil(suite.T(), err) - assert.NotNil(suite.T(), response) - - // valid request - request := &CreateModelRequest{ - Authorization: &AuthorizationDetails{ - SignerAddress: suite.senderAddress.String(), - Message: "__CreateModel", - Signature: suite.getSignature("__CreateModel", 1200, suite.senderPvtKy), - CurrentBlock: 1200, - }, - ModelDetails: &ModelDetails{ - GrpcServiceName: " ", - GrpcMethodName: "/example_service.Calculator/train_add", - Description: "Just Testing", - IsPubliclyAccessible: false, - ModelName: "ABCD", - TrainingDataLink: " ", - AddressList: []string{"A1", "A2", "A3"}, - }, - } - zap.L().Debug("Sender address", zap.Any("value", suite.senderAddress.String())) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2000) - defer cancel() - response, err = suite.service.CreateModel(ctx, request) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), "1", response.ModelDetails.ModelId) - assert.Equal(suite.T(), response.ModelDetails.AddressList, []string{"A1", "A2", "A3", suite.senderAddress.String()}) - userKey := &ModelUserKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - GRPCMethodName: "/example_service.Calculator/train_add", - GRPCServiceName: " ", - UserAddress: suite.senderAddress.String(), - } - //check if we have stored the user's associated model Ids - data, ok, err := suite.service.(*ModelService).userStorage.Get(userKey) - assert.Equal(suite.T(), []string{"1"}, data.ModelIds) - assert.Equal(suite.T(), ok, true) - assert.Nil(suite.T(), err) - - //check if the model Id stored has all the details - key := &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - //GRPCMethodName: "/example_service.Calculator/train_add", - //GRPCServiceName: " ", - ModelId: "1", - } - modelData, ok, err := suite.service.(*ModelService).storage.Get(key) - assert.Equal(suite.T(), []string{"A1", "A2", "A3", suite.senderAddress.String()}, modelData.AuthorizedAddresses) - assert.Equal(suite.T(), ok, true) - assert.Nil(suite.T(), err) - - //send a bad signature - request.Authorization.Signature = suite.getSignature("Different message", 1200, suite.senderPvtKy) - response, err = suite.service.CreateModel(ctx, request) - assert.NotNil(suite.T(), err) - - // valid request - request2 := &CreateModelRequest{ - Authorization: &AuthorizationDetails{ - SignerAddress: suite.senderAddress.String(), - Message: "__CreateModel", - Signature: suite.getSignature("__CreateModel", 1200, suite.senderPvtKy), - CurrentBlock: 1200, - }, - ModelDetails: &ModelDetails{ - GrpcServiceName: " ", - GrpcMethodName: "/example_service.Calculator/train_add", - Description: "Just Testing", - IsPubliclyAccessible: false, - }, - } - zap.L().Debug("Sender address", zap.Any("value", suite.senderAddress.String())) - ctx, cancel = context.WithTimeout(context.Background(), time.Second*2000) - defer cancel() - response, err = suite.service.CreateModel(ctx, request2) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), response.ModelDetails.AddressList, []string{suite.senderAddress.String()}) - //Create an Other Model Id for the same user !!! - response, err = suite.service.CreateModel(ctx, request) - - request3 := &AccessibleModelsRequest{ - GrpcServiceName: " ", - GrpcMethodName: "/example_service.Calculator/train_add", - Authorization: &AuthorizationDetails{ - SignerAddress: suite.senderAddress.String(), - Message: "__UpdateModelAccess", - Signature: suite.getSignature("__UpdateModelAccess", 1200, suite.senderPvtKy), - CurrentBlock: 1200, - }, - } - - ctx, cancel = context.WithTimeout(context.Background(), time.Second*2000) - defer cancel() - response2, err := suite.service.GetAllModels(ctx, request3) - assert.Nil(suite.T(), err) - fmt.Println(response2) - assert.Equal(suite.T(), len(response2.ListOfModels) > 1, true) -} - -func (suite *ModelServiceTestSuite) TestModelService_GetModelStatus() { - request := &ModelDetailsRequest{ - ModelDetails: &ModelDetails{ - ModelId: "1", - GrpcServiceName: " ", - GrpcMethodName: "/example_service.Calculator/train_add", - }, - Authorization: &AuthorizationDetails{ - SignerAddress: suite.senderAddress.String(), - Message: "__GetModelStatus", - Signature: suite.getSignature("__GetModelStatus", 1200, suite.senderPvtKy), - CurrentBlock: 1200, - }, - } - zap.L().Debug("Sender address", zap.Any("value", suite.senderAddress.String())) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2000) - defer cancel() - response, err := suite.service.GetModelStatus(ctx, request) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), Status_IN_PROGRESS, response.Status) -} - -func (suite *ModelServiceTestSuite) TestModelService_UpdateModelAccess() { - - userKey := &ModelUserKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - GRPCMethodName: "/example_service.Calculator/train_add", - GRPCServiceName: " ", - UserAddress: suite.senderAddress.String(), - } - //check if we have stored the user's associated model Ids - err := suite.service.(*ModelService).userStorage.Put(userKey, &ModelUserData{ - ModelIds: []string{"1", "2"}, - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - GRPCMethodName: "/example_service.Calculator/train_add", - GRPCServiceName: " ", - UserAddress: suite.senderAddress.String(), - }) - - modelState, _, _ := suite.service.(*ModelService).userStorage.Get(userKey) - - zap.L().Debug("Model state", zap.Any("value", modelState)) - // assert.Equal(suite.T(), ok, true) - assert.Nil(suite.T(), err) - modelData := &ModelData{ - IsPublic: false, - AuthorizedAddresses: []string{suite.senderAddress.String()}, - Status: Status_IN_PROGRESS, - CreatedByAddress: suite.senderAddress.String(), - ModelId: "1", - UpdatedByAddress: suite.senderAddress.String(), - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - GRPCMethodName: "/example_service.Calculator/train_add", - GRPCServiceName: " ", - Description: "", - IsDefault: false, - TrainingLink: "", - } - - _, err = suite.service.(*ModelService).storage.PutIfAbsent(&ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - GRPCMethodName: "/example_service.Calculator/train_add", - GRPCServiceName: " ", - ModelId: "1", - }, modelData) - // assert.Equal(suite.T(), ok, true) - assert.Nil(suite.T(), err) - - data, _, _ := suite.service.(*ModelService).storage.Get(&ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: suite.service.(*ModelService).organizationMetaData.GetGroupIdString(), - GRPCMethodName: "/example_service.Calculator/train_add", - GRPCServiceName: " ", - ModelId: "1", - }) - - zap.L().Debug("Model data", zap.Any("value", data)) - - request := &UpdateModelRequest{ - UpdateModelDetails: &ModelDetails{ - ModelId: "1", - GrpcServiceName: " ", - GrpcMethodName: "/example_service.Calculator/train_add", - IsPubliclyAccessible: false, - ModelName: "ABCD", - Description: "How are you", - }, - Authorization: &AuthorizationDetails{ - SignerAddress: suite.senderAddress.String(), - Message: "__UpdateModelAccess", - Signature: suite.getSignature("__UpdateModelAccess", 1200, suite.senderPvtKy), - CurrentBlock: 1200, - }, - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2000) - defer cancel() - response, err := suite.service.UpdateModelAccess(ctx, request) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), len(response.GetModelDetails().AddressList), 1) - - //update a model id which is not there - request.UpdateModelDetails.ModelId = "25" - response, err = suite.service.UpdateModelAccess(ctx, request) - assert.NotNil(suite.T(), err) - - //update request with someone who does not have access - request.Authorization.Signature = suite.getSignature("__UpdateModelAccess", 1200, suite.alternateUserPvtKy) - response, err = suite.service.UpdateModelAccess(ctx, request) - assert.NotNil(suite.T(), err) - - //update request with someone who does not have access - request.Authorization = nil - response, err = suite.service.UpdateModelAccess(ctx, request) - assert.NotNil(suite.T(), err) - -} - -func (suite *ModelServiceTestSuite) TestModelService_GetAllModels() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2000) - defer cancel() - request2 := &AccessibleModelsRequest{ - GrpcServiceName: " ", - GrpcMethodName: "/example_service.Calculator/train_add", - Authorization: &AuthorizationDetails{ - SignerAddress: suite.senderAddress.String(), - Message: "__UpdateModelAccess", - Signature: suite.getSignature("__UpdateModelAccess", 1200, suite.alternateUserPvtKy), - CurrentBlock: 1200, - }, - } - response, err := suite.service.GetAllModels(ctx, request2) - assert.NotNil(suite.T(), err) - assert.NotNil(suite.T(), response) - request2.Authorization.Signature = suite.getSignature("__UpdateModelAccess", 1200, suite.alternateUserPvtKy) - response, err = suite.service.GetAllModels(ctx, request2) - assert.NotNil(suite.T(), err) - - request2.Authorization = nil - response, err = suite.service.GetAllModels(ctx, request2) - assert.NotNil(suite.T(), err) - -} - -func (suite *ModelServiceTestSuite) TestModelService_remove() { - sample1 := []string{"a", "b", "c"} - sample2 := []string{"b", "c"} - output := remove(sample1, "a") - assert.Equal(suite.T(), output, sample2) - output = remove(output, "a") - assert.Equal(suite.T(), output, sample2) -} - -func (suite *ModelServiceTestSuite) TestModelService_difference() { - sample1 := []string{"a", "b", "c"} - sample2 := []string{"b", "c", "e", "f"} - output := difference(sample1, sample2) - expected := []string{"a", "e", "f"} - assert.Equal(suite.T(), expected, output) -} - -func (suite *ModelServiceTestSuite) TestModelService_isValuePresent() { - sample1 := []string{"a", "b", "c"} - assert.Equal(suite.T(), isValuePresent("a", sample1), true) - assert.Equal(suite.T(), isValuePresent("d", sample1), false) -} - -func (suite *ModelServiceTestSuite) TestModelService_UDeleteModel() { - response, err := suite.service.DeleteModel(context.Background(), nil) - assert.NotNil(suite.T(), err) - //unauthorized signer - request := &UpdateModelRequest{ - UpdateModelDetails: &ModelDetails{ - ModelId: "1", - GrpcServiceName: " ", - GrpcMethodName: "/example_service.Calculator/train_add", - }, - Authorization: &AuthorizationDetails{ - SignerAddress: suite.alternateUserAddress.String(), - Message: "__GetModelStatus", - Signature: suite.getSignature("__GetModelStatus", 1200, suite.alternateUserPvtKy), - CurrentBlock: 1200, - }, - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2000) - defer cancel() - response, err = suite.service.DeleteModel(ctx, request) - assert.NotNil(suite.T(), err) - - zap.L().Debug("Sender address", zap.Any("value", suite.senderAddress.String())) - //valid signer - request.Authorization.SignerAddress = suite.senderAddress.String() - request.Authorization.Signature = suite.getSignature("__GetModelStatus", 1200, suite.senderPvtKy) - response, err = suite.service.DeleteModel(ctx, request) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), Status_DELETED, response.Status) - - //bad signer - request.Authorization.Message = "blah" - response, err = suite.service.DeleteModel(ctx, request) - assert.NotNil(suite.T(), err) - - request.Authorization = nil - response, err = suite.service.DeleteModel(ctx, request) - assert.NotNil(suite.T(), err) - -} diff --git a/training/tests/functional/service_test.go b/training/tests/functional/service_test.go new file mode 100644 index 00000000..85c0b071 --- /dev/null +++ b/training/tests/functional/service_test.go @@ -0,0 +1,645 @@ +package tests + +import ( + "bytes" + "context" + "crypto/ecdsa" + "math/big" + "slices" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/crypto" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/singnet/snet-daemon/v5/blockchain" + "github.com/singnet/snet-daemon/v5/config" + "github.com/singnet/snet-daemon/v5/storage" + "github.com/singnet/snet-daemon/v5/training" +) + +type DaemonServiceSuite struct { + suite.Suite + modelStorage *training.ModelStorage + userModelStorage *training.ModelUserStorage + pendingModelStorage *training.PendingModelStorage + daemonService training.DaemonServer + unimplementedDaemonService training.DaemonServer + modelKeys []*training.ModelKey + pendingModelKeys []*training.ModelKey // using for checking updated status + grpcServer *grpc.Server + grpcClient *grpc.ClientConn + serviceMetadata *blockchain.ServiceMetadata + organizationMetadata *blockchain.OrganizationMetaData +} + +func TestDaemonServiceSuite(t *testing.T) { + suite.Run(t, new(DaemonServiceSuite)) +} + +var ( + testJsonOrgGroupData = "{ \"org_name\": \"organization_name\", \"org_id\": \"test_org_id\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"license_server_endpoints\": [\"https://licensendpoint:8082\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" + testJsonServiceData = "{ \"version\": 1, \"display_name\": \"Example1\", \"encoding\": \"grpc\", \"service_type\": \"grpc\", \"payment_expiration_threshold\": 40320, \"model_ipfs_hash\": \"Qmdiq8Hu6dYiwp712GtnbBxagyfYyvUY1HYqkH7iN76UCc\", " + + " \"mpe_address\": \"0x7E6366Fbe3bdfCE3C906667911FC5237Cc96BD08\", \"groups\": [ { \"free_calls\": 12, \"free_call_signer_address\": \"0x7DF35C98f41F3Af0df1dc4c7F7D4C19a71Dd059F\", \"endpoints\": [\"http://34.344.33.1:2379\",\"http://34.344.33.1:2389\"], \"group_id\": \"88ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\",\"group_name\": \"default_group\", \"pricing\": [ { \"price_model\": \"fixed_price\", \"price_in_cogs\": 2 }, { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"default\":true, \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] }, { \"endpoints\": [\"http://97.344.33.1:2379\",\"http://67.344.33.1:2389\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"pricing\": [ { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] } ] } " + testUserAddress = "0x3432cBa6BF635Df5fBFD1f1a794fA66D412b8774" +) + +func (suite *DaemonServiceSuite) SetupSuite() { + // setup test config once + suite.setupTestConfig() + + // init service metadata and organization metadata + serviceMetadata, err := blockchain.InitServiceMetaDataFromJson([]byte(testJsonServiceData)) + orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) + if err != nil { + zap.L().Fatal("Error in initinalize organization metadata from json", zap.Error(err)) + } + + suite.serviceMetadata = serviceMetadata + suite.organizationMetadata = orgMetadata + + // setup unimplemented daemon server once + suite.unimplementedDaemonService = training.NoTrainingDaemonServer{} + + // setup test poriver service once + address := "localhost:5001" + suite.grpcServer = startTestService(address) +} + +func (suite *DaemonServiceSuite) SetupTest() { + // setup storages before each test for isolation environment + modelStorage, userModelStorage, pendingModelStorage, publicModelStorage := suite.createTestModels() + suite.modelStorage = modelStorage + suite.userModelStorage = userModelStorage + suite.pendingModelStorage = pendingModelStorage + + suite.daemonService = training.NewTrainingService( + nil, + suite.serviceMetadata, + suite.organizationMetadata, + modelStorage, + userModelStorage, + pendingModelStorage, + publicModelStorage, + ) +} + +func (suite *DaemonServiceSuite) TearDownSuite() { + suite.grpcServer.Stop() +} + +func getTestSignature(text string, blockNumber int, privateKey *ecdsa.PrivateKey) (signature []byte) { + HashPrefix32Bytes := []byte("\x19Ethereum Signed Message:\n32") + + message := bytes.Join([][]byte{ + []byte(text), + crypto.PubkeyToAddress(privateKey.PublicKey).Bytes(), + math.U256Bytes(big.NewInt(int64(blockNumber))), + }, nil) + + hash := crypto.Keccak256( + HashPrefix32Bytes, + crypto.Keccak256(message), + ) + + signature, err := crypto.Sign(hash, privateKey) + if err != nil { + zap.L().Fatal("Cannot sign test message", zap.Error(err)) + } + + return signature +} + +func createTestAuthDetails() *training.AuthorizationDetails { + privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") + if err != nil { + zap.L().Fatal("error in creating private key", zap.Error(err)) + } + return &training.AuthorizationDetails{ + CurrentBlock: 0, + Message: "__CreateModel", + Signature: getTestSignature("__CreateModel", 0, privateKey), + SignerAddress: "0x3432cBa6BF635Df5fBFD1f1a794fA66D412b8774", + } +} + +func creatBadTestAuthDetails() *training.AuthorizationDetails { + privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") + if err != nil { + zap.L().Fatal("error in creating private key", zap.Error(err)) + } + return &training.AuthorizationDetails{ + CurrentBlock: 0, + Message: "badMessage", + Signature: getTestSignature("badMessage", 0, privateKey), + SignerAddress: "0x4444cBa6BF635Df5fBFD1f1a794fA66D412b8774", + } +} + +func (suite *DaemonServiceSuite) setupTestConfig() { + testConfigJson := ` +{ + "blockchain_enabled": true, + "blockchain_network_selected": "sepolia", + "daemon_end_point": "127.0.0.1:8080", + "daemon_group_name":"default_group", + "payment_channel_storage_type": "etcd", + "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "ipfs_timeout" : 30, + "passthrough_enabled": true, + "passthrough_endpoint":"http://127.0.0.1:5002", + "service_id": "service_id", + "organization_id": "test_org_id", + "metering_enabled": false, + "ssl_cert": "", + "ssl_key": "", + "max_message_size_in_mb" : 4, + "daemon_type": "grpc", + "enable_dynamic_pricing":false, + "allowed_user_flag" :false, + "auto_ssl_domain": "", + "auto_ssl_cache_dir": ".certs", + "private_key": "", + "log": { + "level": "info", + "timezone": "UTC", + "formatter": { + "type": "text", + "timestamp_format": "2006-01-02T15:04:05.999Z07:00" + }, + "output": { + "type": ["file", "stdout"], + "file_pattern": "./snet-daemon.%Y%m%d.log", + "current_link": "./snet-daemon.log", + "max_size_in_mb": 10, + "max_age_in_days": 7, + "rotation_count": 0 + }, + "hooks": [] + }, + "model_maintenance_endpoint": "http://localhost:5001", + "payment_channel_storage_client": { + "connection_timeout": "0s", + "request_timeout": "0s", + "hot_reload": true + }, + "payment_channel_storage_server": { + "id": "storage-1", + "scheme": "http", + "host" : "127.0.0.1", + "client_port": 2379, + "peer_port": 2380, + "token": "unique-token", + "cluster": "storage-1=http://127.0.0.1:2380", + "startup_timeout": "1m", + "data_dir": "storage-data-dir-1.etcd", + "log_level": "info", + "log_outputs": ["./etcd-server.log"], + "enabled": false + }, + "alerts_email": "", + "service_heartbeat_type": "http", + "token_expiry_in_minutes": 1440, + "model_training_enabled": false +}` + + var testConfig = viper.New() + err := config.ReadConfigFromJsonString(testConfig, testConfigJson) + if err != nil { + zap.L().Fatal("Error in reading config") + } + + config.SetVip(testConfig) +} + +func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage, *training.PublicModelStorage) { + memStorage := storage.NewMemStorage() + modelStorage := training.NewModelStorage(memStorage) + userModelStorage := training.NewUserModelStorage(memStorage) + pendingModelStorage := training.NewPendingModelStorage(memStorage) + publicModelStorage := training.NewPublicModelStorage(memStorage) + + modelA := &training.ModelData{ + IsPublic: true, + ModelName: "testModel", + AuthorizedAddresses: []string{}, + Status: training.Status_VALIDATING, + CreatedByAddress: "address", + ModelId: "test_1", + UpdatedByAddress: "string", + GroupId: "string", + OrganizationId: "string", + ServiceId: "string", + GRPCMethodName: "string", + GRPCServiceName: "string", + Description: "string", + IsDefault: true, + TrainingLink: "string", + UpdatedDate: "string", + } + + modelAKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: "test_1", + } + + modelB := &training.ModelData{ + IsPublic: false, + ModelName: "testModel", + AuthorizedAddresses: []string{}, + Status: training.Status_CREATED, + CreatedByAddress: "address", + ModelId: "test_2", + UpdatedByAddress: "string", + GroupId: "string", + OrganizationId: "string", + ServiceId: "string", + GRPCMethodName: "string", + GRPCServiceName: "string", + Description: "string", + IsDefault: true, + TrainingLink: "string", + UpdatedDate: "string", + } + + modelBKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: "test_2", + } + + modelC := &training.ModelData{ + IsPublic: false, + ModelName: "testModel", + AuthorizedAddresses: []string{}, + Status: training.Status_CREATED, + CreatedByAddress: "address", + ModelId: "test_3", + UpdatedByAddress: "string", + GroupId: "string", + OrganizationId: "string", + ServiceId: "string", + GRPCMethodName: "string", + GRPCServiceName: "string", + Description: "string", + IsDefault: true, + TrainingLink: "string", + UpdatedDate: "string", + } + + modelCKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: "test_3", + } + + modelStorage.Put(modelAKey, modelA) + modelStorage.Put(modelBKey, modelB) + modelStorage.Put(modelCKey, modelC) + + // adding to user models sotrage + userModelKey := &training.ModelUserKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + UserAddress: testUserAddress, + } + + userModelData := &training.ModelUserData{ + ModelIds: []string{"test_3"}, + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + UserAddress: testUserAddress, + } + + userModelStorage.Put(userModelKey, userModelData) + + // adding to pending models storage + pendingModelKey := &training.PendingModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + } + + pendingModelsData := &training.PendingModelData{ + ModelIDs: []string{"test_1"}, + } + + pendingModelStorage.Put(pendingModelKey, pendingModelsData) + + // adding to public models storage + publicModelKey := &training.PublicModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + } + + publicModelsData := &training.PublicModelData{ + ModelIDs: []string{"test_1"}, + } + + publicModelStorage.Put(publicModelKey, publicModelsData) + + // setup keys in suite + suite.modelKeys = []*training.ModelKey{modelAKey, modelBKey, modelCKey} + suite.pendingModelKeys = []*training.ModelKey{modelAKey} + + // return all model keys, storages + return modelStorage, userModelStorage, pendingModelStorage, publicModelStorage +} + +func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, authDetails *training.AuthorizationDetails) string { + newModel := &training.NewModel{ + Name: modelName, + Description: "test_desc", + GrpcMethodName: "test_grpc_method_name", + GrpcServiceName: "test_grpc_service_name", + AddressList: []string{}, + IsPublic: false, + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + } + + request := &training.NewModelRequest{ + Authorization: authDetails, + Model: newModel, + } + response, err := suite.daemonService.CreateModel(context.Background(), request) + if err != nil { + zap.L().Fatal("error in creating additional test model", zap.Error(err)) + } + + return response.ModelId +} + +func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { + testAuthCreads := createTestAuthDetails() + badTestAuthCreads := creatBadTestAuthDetails() + + // check without request + response1, err := suite.daemonService.GetModel(context.Background(), nil) + assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) + + // check without auth + request2 := &training.CommonRequest{ + Authorization: nil, + ModelId: "test_2", + } + response2, err := suite.daemonService.GetModel(context.Background(), request2) + assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response2.Status) + + // check with bad auth + request3 := &training.CommonRequest{ + Authorization: badTestAuthCreads, + ModelId: "test_2", + } + response3, err := suite.daemonService.GetModel(context.Background(), request3) + assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) + + // check modelId is not empty string + request4 := &training.CommonRequest{ + Authorization: testAuthCreads, + ModelId: "", + } + response4, err := suite.daemonService.GetModel(context.Background(), request4) + assert.NotNil(suite.T(), err) + assert.ErrorContains(suite.T(), err, training.ErrEmptyModelID.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) + + // check without access to model + request5 := &training.CommonRequest{ + Authorization: testAuthCreads, + ModelId: "test_2", + } + response5, err := suite.daemonService.GetModel(context.Background(), request5) + assert.ErrorContains(suite.T(), err, training.ErrAccessToModel.Error()) + assert.Equal(suite.T(), &training.ModelResponse{}, response5) + + // check access to public model + request6 := &training.CommonRequest{ + Authorization: testAuthCreads, + ModelId: "test_1", + } + response6, err := suite.daemonService.GetModel(context.Background(), request6) + assert.Nil(suite.T(), err) + assert.NotEmpty(suite.T(), response6) + assert.Equal(suite.T(), true, response6.IsPublic) + + //check access to non public model + request7 := &training.CommonRequest{ + Authorization: testAuthCreads, + ModelId: "test_3", + } + response7, err := suite.daemonService.GetModel(context.Background(), request7) + assert.Nil(suite.T(), err) + assert.NotEmpty(suite.T(), response7) + assert.Equal(suite.T(), false, response7.IsPublic) +} + +func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { + testAuthCreads := createTestAuthDetails() + badTestAuthCreads := creatBadTestAuthDetails() + + newModel := &training.NewModel{ + Name: "new_test_model", + Description: "test_desc", + GrpcMethodName: "test_grpc_method_name", + GrpcServiceName: "test_grpc_service_name", + AddressList: []string{}, + IsPublic: false, + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + } + + newEmptyModel := &training.NewModel{} + + // check without request + response1, err := suite.daemonService.CreateModel(context.Background(), nil) + assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) + + // check without auth + request2 := &training.NewModelRequest{ + Authorization: nil, + Model: newModel, + } + response2, err := suite.daemonService.CreateModel(context.Background(), request2) + assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response2.Status) + + // check with bad auth + request3 := &training.NewModelRequest{ + Authorization: badTestAuthCreads, + Model: newModel, + } + response3, err := suite.daemonService.CreateModel(context.Background(), request3) + assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) + + // check with emptyModel + request4 := &training.NewModelRequest{ + Authorization: testAuthCreads, + Model: newEmptyModel, + } + response4, err := suite.daemonService.CreateModel(context.Background(), request4) + assert.ErrorContains(suite.T(), err, training.ErrNoGRPCServiceOrMethod.Error()) + assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) + + // check with auth + request5 := &training.NewModelRequest{ + Authorization: testAuthCreads, + Model: newModel, + } + response5, err := suite.daemonService.CreateModel(context.Background(), request5) + assert.Nil(suite.T(), err) + assert.Equal(suite.T(), training.Status_CREATED, response5.Status) + + // check model creation in model storage + modelKey := &training.ModelKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + ModelId: response5.ModelId, + } + + modelData, ok, err := suite.modelStorage.Get(modelKey) + assert.Nil(suite.T(), err) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), response5.ModelId, modelData.ModelId) + assert.Equal(suite.T(), newModel.Name, modelData.ModelName) + + // check user model data creation in user model storage + userModelKey := &training.ModelUserKey{ + OrganizationId: "test_org_id", + ServiceId: "service_id", + GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", + UserAddress: testUserAddress, + } + + userModelData, ok, err := suite.userModelStorage.Get(userModelKey) + assert.Nil(suite.T(), err) + assert.True(suite.T(), ok) + assert.True(suite.T(), slices.Contains(userModelData.ModelIds, response5.ModelId)) +} + +func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { + testAuthCreads := createTestAuthDetails() + badTestAuthCreads := creatBadTestAuthDetails() + + newAdditionalTestModelId := suite.createAdditionalTestModel("new_additional_test_model", testAuthCreads) + + expectedModelIds := []string{"test_3", newAdditionalTestModelId, "test_1"} + + // check without request + response1, err := suite.daemonService.GetAllModels(context.Background(), nil) + assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.Nil(suite.T(), response1.ListOfModels) + + // check without auth + request2 := &training.AllModelsRequest{ + Authorization: nil, + } + response2, err := suite.daemonService.GetAllModels(context.Background(), request2) + assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.Nil(suite.T(), response2.ListOfModels) + + // check with bad auth + request3 := &training.AllModelsRequest{ + Authorization: badTestAuthCreads, + } + response3, err := suite.daemonService.GetAllModels(context.Background(), request3) + assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) + assert.Nil(suite.T(), response3.ListOfModels) + + // check with auth and without filters + request4 := &training.AllModelsRequest{ + Authorization: testAuthCreads, + } + response4, err := suite.daemonService.GetAllModels(context.Background(), request4) + assert.Nil(suite.T(), err) + modelIds := []string{} + for _, model := range response4.ListOfModels { + modelIds = append(modelIds, model.ModelId) + } + + assert.True(suite.T(), slices.Equal(expectedModelIds, modelIds)) +} + +func (suite *DaemonServiceSuite) TestDaemonSerice_ManageUpdateStatusWorkers() { + duration := time.Second * 10 + deadline := time.Now().Add(duration) + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + select { + case <-ctx.Done(): + zap.L().Info("Context done", zap.Error(ctx.Err())) + case <-time.After(duration): + zap.L().Info("Operation timed out after", zap.Duration("duration", duration)) + } + + for _, modelKey := range suite.pendingModelKeys { + modelData, ok, err := suite.modelStorage.Get(modelKey) + assert.Nil(suite.T(), err) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), training.Status_VALIDATED, modelData.Status) + } +} + +func (suite *DaemonServiceSuite) TestDaemonService_UnimplementedDaemonService() { + response1, err := suite.unimplementedDaemonService.CreateModel(context.TODO(), &training.NewModelRequest{}) + assert.NotNil(suite.T(), err) + assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) + + _, err = suite.unimplementedDaemonService.ValidateModelPrice(context.TODO(), &training.AuthValidateRequest{}) + assert.NotNil(suite.T(), err) + + response2, err := suite.unimplementedDaemonService.ValidateModel(context.TODO(), &training.AuthValidateRequest{}) + assert.NotNil(suite.T(), err) + assert.Equal(suite.T(), training.Status_ERRORED, response2.Status) + + _, err = suite.unimplementedDaemonService.TrainModelPrice(context.TODO(), &training.CommonRequest{}) + assert.NotNil(suite.T(), err) + + response3, err := suite.unimplementedDaemonService.TrainModel(context.TODO(), &training.CommonRequest{}) + assert.NotNil(suite.T(), err) + assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) + + response4, err := suite.unimplementedDaemonService.DeleteModel(context.TODO(), &training.CommonRequest{}) + assert.NotNil(suite.T(), err) + assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) + + response5, err := suite.unimplementedDaemonService.GetAllModels(context.TODO(), &training.AllModelsRequest{}) + assert.NotNil(suite.T(), err) + assert.Equal(suite.T(), []*training.ModelResponse{}, response5.ListOfModels) + + response6, err := suite.unimplementedDaemonService.GetModel(context.TODO(), &training.CommonRequest{}) + assert.NotNil(suite.T(), err) + assert.Equal(suite.T(), training.Status_ERRORED, response6.Status) + + response7, err := suite.unimplementedDaemonService.UpdateModel(context.Background(), &training.UpdateModelRequest{}) + assert.NotNil(suite.T(), err) + assert.Equal(suite.T(), training.Status_ERRORED, response7.Status) + + _, err = suite.unimplementedDaemonService.GetTrainingMetadata(context.Background(), &emptypb.Empty{}) + assert.NotNil(suite.T(), err) + + _, err = suite.unimplementedDaemonService.GetMethodMetadata(context.TODO(), &training.MethodMetadataRequest{}) + assert.NotNil(suite.T(), err) +} diff --git a/training/tests/integration/test_provider_service.go b/training/tests/functional/test_provider_service.go similarity index 100% rename from training/tests/integration/test_provider_service.go rename to training/tests/functional/test_provider_service.go diff --git a/training/tests/integration/daemon_service_test.go b/training/tests/integration/daemon_service_test.go deleted file mode 100644 index 5d650a2d..00000000 --- a/training/tests/integration/daemon_service_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package tests - -import ( - "context" - "testing" - "time" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "go.uber.org/zap" - "google.golang.org/grpc" - - "github.com/singnet/snet-daemon/v5/blockchain" - "github.com/singnet/snet-daemon/v5/config" - "github.com/singnet/snet-daemon/v5/storage" - "github.com/singnet/snet-daemon/v5/training" -) - -type DaemonServiceSuite struct { - suite.Suite - modelStorage *training.ModelStorage - userModelStorage *training.ModelUserStorage - pendingModelStorage *training.PendingModelStorage - daemonService training.DaemonServer - modelKeys []*training.ModelKey - grpcServer *grpc.Server -} - -var ( - testJsonOrgGroupData = "{ \"org_name\": \"organization_name\", \"org_id\": \"test_org_id\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"license_server_endpoints\": [\"https://licensendpoint:8082\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" - testJsonServiceData = "{ \"version\": 1, \"display_name\": \"Example1\", \"encoding\": \"grpc\", \"service_type\": \"grpc\", \"payment_expiration_threshold\": 40320, \"model_ipfs_hash\": \"Qmdiq8Hu6dYiwp712GtnbBxagyfYyvUY1HYqkH7iN76UCc\", " + - " \"mpe_address\": \"0x7E6366Fbe3bdfCE3C906667911FC5237Cc96BD08\", \"groups\": [ { \"free_calls\": 12, \"free_call_signer_address\": \"0x7DF35C98f41F3Af0df1dc4c7F7D4C19a71Dd059F\", \"endpoints\": [\"http://34.344.33.1:2379\",\"http://34.344.33.1:2389\"], \"group_id\": \"88ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\",\"group_name\": \"default_group\", \"pricing\": [ { \"price_model\": \"fixed_price\", \"price_in_cogs\": 2 }, { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"default\":true, \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] }, { \"endpoints\": [\"http://97.344.33.1:2379\",\"http://67.344.33.1:2389\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"pricing\": [ { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] } ] } " -) - -func (suite *DaemonServiceSuite) SetupSuite() { - suite.setupTestConfig() - - modelKeys, modelStorage, userModelStorage, pendingModelStorage, publicModelStorage := suite.createTestModels() - suite.modelKeys = modelKeys - suite.modelStorage = modelStorage - suite.userModelStorage = userModelStorage - suite.pendingModelStorage = pendingModelStorage - - serviceMetadata, err := blockchain.InitServiceMetaDataFromJson([]byte(testJsonServiceData)) - orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) - if err != nil { - zap.L().Fatal("Error in initinalize organization metadata from json", zap.Error(err)) - } - - suite.daemonService = training.NewTrainingService( - nil, - serviceMetadata, - orgMetadata, - modelStorage, - userModelStorage, - pendingModelStorage, - publicModelStorage, - ) - - address := "localhost:5001" - suite.grpcServer = startTestService(address) -} - -func TestDaemonServiceSuite(t *testing.T) { - suite.Run(t, new(DaemonServiceSuite)) -} - -func (suite *DaemonServiceSuite) setupTestConfig() { - testConfigJson := ` -{ - "blockchain_enabled": true, - "blockchain_network_selected": "sepolia", - "daemon_end_point": "127.0.0.1:8080", - "daemon_group_name":"default_group", - "payment_channel_storage_type": "etcd", - "ipfs_end_point": "http://ipfs.singularitynet.io:80", - "ipfs_timeout" : 30, - "passthrough_enabled": true, - "passthrough_endpoint":"http://127.0.0.1:5002", - "service_id": "service_id", - "organization_id": "test_org_id", - "metering_enabled": false, - "ssl_cert": "", - "ssl_key": "", - "max_message_size_in_mb" : 4, - "daemon_type": "grpc", - "enable_dynamic_pricing":false, - "allowed_user_flag" :false, - "auto_ssl_domain": "", - "auto_ssl_cache_dir": ".certs", - "private_key": "", - "log": { - "level": "info", - "timezone": "UTC", - "formatter": { - "type": "text", - "timestamp_format": "2006-01-02T15:04:05.999Z07:00" - }, - "output": { - "type": ["file", "stdout"], - "file_pattern": "./snet-daemon.%Y%m%d.log", - "current_link": "./snet-daemon.log", - "max_size_in_mb": 10, - "max_age_in_days": 7, - "rotation_count": 0 - }, - "hooks": [] - }, - "model_maintenance_endpoint": "http://localhost:5001", - "payment_channel_storage_client": { - "connection_timeout": "0s", - "request_timeout": "0s", - "hot_reload": true - }, - "payment_channel_storage_server": { - "id": "storage-1", - "scheme": "http", - "host" : "127.0.0.1", - "client_port": 2379, - "peer_port": 2380, - "token": "unique-token", - "cluster": "storage-1=http://127.0.0.1:2380", - "startup_timeout": "1m", - "data_dir": "storage-data-dir-1.etcd", - "log_level": "info", - "log_outputs": ["./etcd-server.log"], - "enabled": false - }, - "alerts_email": "", - "service_heartbeat_type": "http", - "token_expiry_in_minutes": 1440, - "model_training_enabled": false -}` - - var testConfig = viper.New() - err := config.ReadConfigFromJsonString(testConfig, testConfigJson) - if err != nil { - zap.L().Fatal("Error in reading config") - } - - config.SetVip(testConfig) -} - -func (suite *DaemonServiceSuite) createTestModels() ([]*training.ModelKey, - *training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage, *training.PublicModelStorage) { - memStorage := storage.NewMemStorage() - modelStorage := training.NewModelStorage(memStorage) - userModelStorage := training.NewUserModelStorage(memStorage) - pendingModelStorage := training.NewPendingModelStorage(memStorage) - publicModelStorage := training.NewPublicModelStorage(memStorage) - - modelA := &training.ModelData{ - IsPublic: true, - ModelName: "testModel", - AuthorizedAddresses: []string{}, - Status: training.Status_CREATED, - CreatedByAddress: "address", - ModelId: "1", - UpdatedByAddress: "string", - GroupId: "string", - OrganizationId: "string", - ServiceId: "string", - GRPCMethodName: "string", - GRPCServiceName: "string", - Description: "string", - IsDefault: true, - TrainingLink: "string", - UpdatedDate: "string", - } - - modelAKey := &training.ModelKey{ - OrganizationId: "test_org_id", - ServiceId: "service_id", - GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - ModelId: "1", - } - - modelB := &training.ModelData{ - IsPublic: true, - ModelName: "testModel", - AuthorizedAddresses: []string{}, - Status: training.Status_CREATED, - CreatedByAddress: "address", - ModelId: "1", - UpdatedByAddress: "string", - GroupId: "string", - OrganizationId: "string", - ServiceId: "string", - GRPCMethodName: "string", - GRPCServiceName: "string", - Description: "string", - IsDefault: true, - TrainingLink: "string", - UpdatedDate: "string", - } - - modelBKey := &training.ModelKey{ - OrganizationId: "test_org_id", - ServiceId: "service_id", - GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - ModelId: "2", - } - - modelStorage.Put(modelAKey, modelA) - modelStorage.Put(modelBKey, modelB) - modelKeys := []*training.ModelKey{modelAKey, modelBKey} - - key := &training.PendingModelKey{ - OrganizationId: "test_org_id", - ServiceId: "service_id", - GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - } - - pendingModelsData := &training.PendingModelData{ - ModelIDs: []string{"1", "2"}, - } - - pendingModelStorage.Put(key, pendingModelsData) - - return modelKeys, modelStorage, userModelStorage, pendingModelStorage, publicModelStorage -} - -func (suite *DaemonServiceSuite) TestRunDaemonService() { - - duration := time.Second * 10 - deadline := time.Now().Add(duration) - ctx, cancel := context.WithDeadline(context.Background(), deadline) - defer cancel() - - select { - case <-ctx.Done(): - zap.L().Info("Context done", zap.Error(ctx.Err())) - case <-time.After(duration): - zap.L().Info("Operation timed out after", zap.Duration("duration", duration)) - } - - for _, modelKey := range suite.modelKeys { - modelData, ok, err := suite.modelStorage.Get(modelKey) - assert.Nil(suite.T(), err) - assert.True(suite.T(), ok) - assert.Equal(suite.T(), training.Status_VALIDATED, modelData.Status) - } -} diff --git a/training/util.go b/training/util.go index 2a9a3574..5a3310fa 100644 --- a/training/util.go +++ b/training/util.go @@ -2,10 +2,11 @@ package training import ( "bytes" + "math/big" + "github.com/ethereum/go-ethereum/common/math" "github.com/singnet/snet-daemon/v5/authutils" "github.com/singnet/snet-daemon/v5/utils" - "math/big" ) // message used to sign is of the form ("__create_model", mpe_address, current_block_number) From 216ed50ebdef876c672be30de33b5ee1d68d4830 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Fri, 24 Jan 2025 13:26:48 +0300 Subject: [PATCH 12/22] add pay interceptors for training & fix streams --- authutils/auth_service.go | 2 +- escrow/allowed_user_payment_handler.go | 2 +- escrow/free_call_payment_handler.go | 2 +- escrow/income.go | 100 ++++++++-- escrow/payment_handler.go | 6 +- escrow/prepaid_handler.go | 2 +- escrow/train_pay_handler.go | 186 ++++++++++++++++++ go.mod | 14 +- go.sum | 16 ++ ...{interceptors.go => stream_interceptor.go} | 36 ++-- handler/unary_interceptor.go | 147 ++++++++++++++ pricing/pricing_strategy.go | 8 +- snetd/cmd/components.go | 97 ++++++--- snetd/cmd/serve.go | 3 +- training/service.go | 139 ++++++------- training/storage.go | 35 +++- .../tests/integration/daemon_service_test.go | 2 +- 17 files changed, 650 insertions(+), 147 deletions(-) create mode 100644 escrow/train_pay_handler.go rename handler/{interceptors.go => stream_interceptor.go} (93%) create mode 100644 handler/unary_interceptor.go diff --git a/authutils/auth_service.go b/authutils/auth_service.go index 655624b2..581f0978 100755 --- a/authutils/auth_service.go +++ b/authutils/auth_service.go @@ -150,7 +150,7 @@ func GetSignature(message []byte, privateKey *ecdsa.PrivateKey) (signature []byt signature, err := crypto.Sign(hash, privateKey) if err != nil { - panic(fmt.Sprintf("Cannot sign test message: %v", err)) + zap.L().Fatal(fmt.Sprintf("Cannot sign test message: %v", err)) } return signature diff --git a/escrow/allowed_user_payment_handler.go b/escrow/allowed_user_payment_handler.go index afdc0c37..205bcae0 100644 --- a/escrow/allowed_user_payment_handler.go +++ b/escrow/allowed_user_payment_handler.go @@ -11,7 +11,7 @@ type allowedUserPaymentHandler struct { validator *AllowedUserPaymentValidator } -func AllowedUserPaymentHandler() handler.PaymentHandler { +func AllowedUserPaymentHandler() handler.StreamPaymentHandler { return &allowedUserPaymentHandler{ validator: &AllowedUserPaymentValidator{}, } diff --git a/escrow/free_call_payment_handler.go b/escrow/free_call_payment_handler.go index 97ad6713..b09827f6 100644 --- a/escrow/free_call_payment_handler.go +++ b/escrow/free_call_payment_handler.go @@ -23,7 +23,7 @@ type freeCallPaymentHandler struct { // NewPaymentHandler returns new MultiPartyEscrow contract payment handler. func FreeCallPaymentHandler( freeCallService FreeCallUserService, processor *blockchain.Processor, metadata *blockchain.OrganizationMetaData, - pServiceMetaData *blockchain.ServiceMetadata) handler.PaymentHandler { + pServiceMetaData *blockchain.ServiceMetadata) handler.StreamPaymentHandler { return &freeCallPaymentHandler{ service: freeCallService, orgMetadata: metadata, diff --git a/escrow/income.go b/escrow/income.go index 91720fd2..b31cc79b 100644 --- a/escrow/income.go +++ b/escrow/income.go @@ -1,16 +1,19 @@ package escrow import ( + "errors" "github.com/singnet/snet-daemon/v5/pricing" + "github.com/singnet/snet-daemon/v5/training" + "go.uber.org/zap" "math/big" "github.com/singnet/snet-daemon/v5/handler" ) -// IncomeData is used to pass information to the pricing validation system. +// IncomeStreamData is used to pass information to the pricing validation system. // This system can use information about call to calculate price and verify // income received. -type IncomeData struct { +type IncomeStreamData struct { // Income is a difference between previous authorized amount and amount // which was received with current call. Income *big.Int @@ -19,33 +22,108 @@ type IncomeData struct { GrpcContext *handler.GrpcStreamContext } -// IncomeValidator uses pricing information to check that call was paid +// IncomeStreamValidator uses pricing information to check that call was paid // correctly by channel sender. This interface can be implemented differently // depending on pricing policy. For instance one can verify that call is paid // according to invoice. Each RPC method can have different price and so on. To // implement these strategies additional information from gRPC context can be // required. In such case it should be added into handler.GrpcStreamContext. -type IncomeValidator interface { +type IncomeStreamValidator interface { // Validate returns nil if validation is successful or correct PaymentError // status to be sent to client in case of validation error. - Validate(*IncomeData) (err error) + Validate(*IncomeStreamData) (err error) } -type incomeValidator struct { +type incomeStreamValidator struct { priceStrategy *pricing.PricingStrategy + storage *training.ModelStorage +} + +// NewIncomeStreamValidator returns new income validator instance +func NewIncomeStreamValidator(pricing *pricing.PricingStrategy, storage *training.ModelStorage) (validator IncomeStreamValidator) { + return &incomeStreamValidator{priceStrategy: pricing, storage: storage} +} + +func (validator *incomeStreamValidator) Validate(data *IncomeStreamData) (err error) { + + price := big.NewInt(0) + + if data.GrpcContext.Info.FullMethod == "/training_daemon.Daemon/upload_and_validate" { + modelID, ok := data.GrpcContext.MD[handler.TrainingModelId] + if !ok { + return errors.New("no training model found") + } + + model, err := validator.storage.GetModel(modelID[0]) + if err != nil { + return errors.New("no training model found") + } + + price = price.SetUint64(model.ValidatePrice) + } else { + price, err = validator.priceStrategy.GetPrice(data.GrpcContext) + if err != nil { + return err + } + } + + if data.Income.Cmp(price) != 0 { + err = NewPaymentError(Unauthenticated, "income %d does not equal to price %d", data.Income, price) + return + } + + return +} + +type trainUnaryValidator struct { + priceStrategy *pricing.PricingStrategy + storage *training.ModelStorage +} + +type IncomeUnaryValidator interface { + // Validate returns nil if validation is successful or correct PaymentError + // status to be sent to client in case of validation error. + Validate(data *IncomeUnaryData) (err error) +} + +type IncomeUnaryData struct { + // Income is a difference between previous authorized amount and amount + // which was received with current call. + Income *big.Int + // GrpcContext contains gRPC stream context information. For instance + // metadata could be used to pass invoice id to check pricing. + GrpcContext *handler.GrpcUnaryContext } // NewIncomeValidator returns new income validator instance -func NewIncomeValidator(pricing *pricing.PricingStrategy) (validator IncomeValidator) { - return &incomeValidator{priceStrategy: pricing} +func NewTrainValidator(storage *training.ModelStorage) (validator IncomeUnaryValidator) { + return &trainUnaryValidator{storage: storage} } -func (validator *incomeValidator) Validate(data *IncomeData) (err error) { - price, err := validator.priceStrategy.GetPrice(data.GrpcContext) +func (validator *trainUnaryValidator) Validate(data *IncomeUnaryData) (err error) { + modelID, ok := data.GrpcContext.MD[handler.TrainingModelId] + if !ok { + return errors.New("no training model found") + } + + model, err := validator.storage.GetModel(modelID[0]) if err != nil { - return err + return errors.New("no training model found") } + price := big.NewInt(0) + + switch data.GrpcContext.Info.FullMethod { + case "/training_daemon.Daemon/train_model": + price = price.SetUint64(model.TrainPrice) + case "/training_daemon.Daemon/validate_model": + price = price.SetUint64(model.ValidatePrice) + default: + return nil + } + + zap.L().Debug("[Validate]", zap.Uint64("price", price.Uint64())) + if data.Income.Cmp(price) != 0 { err = NewPaymentError(Unauthenticated, "income %d does not equal to price %d", data.Income, price) return diff --git a/escrow/payment_handler.go b/escrow/payment_handler.go index b3bccb19..98184e9b 100644 --- a/escrow/payment_handler.go +++ b/escrow/payment_handler.go @@ -23,14 +23,14 @@ const ( type paymentChannelPaymentHandler struct { service PaymentChannelService mpeContractAddress func() common.Address - incomeValidator IncomeValidator + incomeValidator IncomeStreamValidator } // NewPaymentHandler returns new MultiPartyEscrow contract payment handler. func NewPaymentHandler( service PaymentChannelService, processor *blockchain.Processor, - incomeValidator IncomeValidator) handler.PaymentHandler { + incomeValidator IncomeStreamValidator) handler.StreamPaymentHandler { return &paymentChannelPaymentHandler{ service: service, mpeContractAddress: processor.EscrowContractAddress, @@ -55,7 +55,7 @@ func (h *paymentChannelPaymentHandler) Payment(context *handler.GrpcStreamContex income := big.NewInt(0) income.Sub(internalPayment.Amount, transaction.Channel().AuthorizedAmount) - e = h.incomeValidator.Validate(&IncomeData{Income: income, GrpcContext: context}) + e = h.incomeValidator.Validate(&IncomeStreamData{Income: income, GrpcContext: context}) if e != nil { //Make sure the transaction is Rolled back , else this will cause a lock on the channel transaction.Rollback() diff --git a/escrow/prepaid_handler.go b/escrow/prepaid_handler.go index 8bfb348c..5493c958 100644 --- a/escrow/prepaid_handler.go +++ b/escrow/prepaid_handler.go @@ -40,7 +40,7 @@ func (validator *PrePaidPaymentValidator) Validate(payment *PrePaidPayment) (err // NewPaymentHandler returns new MultiPartyEscrow contract payment handler. func NewPrePaidPaymentHandler( PrePaidService PrePaidService, metadata *blockchain.OrganizationMetaData, - pServiceMetaData *blockchain.ServiceMetadata, pricing *pricing.PricingStrategy, manager token.Manager) handler.PaymentHandler { + pServiceMetaData *blockchain.ServiceMetadata, pricing *pricing.PricingStrategy, manager token.Manager) handler.StreamPaymentHandler { return &PrePaidPaymentHandler{ service: PrePaidService, orgMetadata: metadata, diff --git a/escrow/train_pay_handler.go b/escrow/train_pay_handler.go new file mode 100644 index 00000000..79a1cfcc --- /dev/null +++ b/escrow/train_pay_handler.go @@ -0,0 +1,186 @@ +package escrow + +import ( + "github.com/ethereum/go-ethereum/common" + "google.golang.org/grpc/metadata" + "math/big" + + "github.com/singnet/snet-daemon/v5/blockchain" + "github.com/singnet/snet-daemon/v5/handler" +) + +const ( + TrainPaymentType = "train-call" +) + +type trainUnaryPaymentHandler struct { + service PaymentChannelService + mpeContractAddress func() common.Address + incomeValidator IncomeUnaryValidator +} + +type trainStreamPaymentHandler struct { + service PaymentChannelService + mpeContractAddress func() common.Address + incomeValidator IncomeStreamValidator +} + +func (t trainStreamPaymentHandler) Type() (typ string) { + return TrainPaymentType +} + +func (t trainStreamPaymentHandler) Payment(context *handler.GrpcStreamContext) (payment handler.Payment, err *handler.GrpcError) { + internalPayment, err := t.getPaymentFromContext(context.MD) + if err != nil { + return + } + + transaction, e := t.service.StartPaymentTransaction(internalPayment) + if e != nil { + return nil, paymentErrorToGrpcError(e) + } + + income := big.NewInt(0) + income.Sub(internalPayment.Amount, transaction.Channel().AuthorizedAmount) + e = t.incomeValidator.Validate(&IncomeStreamData{Income: income, GrpcContext: context}) + if e != nil { + //Make sure the transaction is Rolled back , else this will cause a lock on the channel + transaction.Rollback() + return nil, paymentErrorToGrpcError(e) + } + + return transaction, nil +} + +func (t trainStreamPaymentHandler) Complete(payment handler.Payment) (err *handler.GrpcError) { + if err = paymentErrorToGrpcError(payment.(*paymentTransaction).Commit()); err == nil { + PublishChannelStats(payment) + } + return err +} + +func (t trainStreamPaymentHandler) CompleteAfterError(payment handler.Payment, result error) (err *handler.GrpcError) { + return paymentErrorToGrpcError(payment.(*paymentTransaction).Rollback()) +} + +func (t trainStreamPaymentHandler) getPaymentFromContext(md metadata.MD) (payment *Payment, err *handler.GrpcError) { + channelID, err := handler.GetBigInt(md, handler.PaymentChannelIDHeader) + if err != nil { + return + } + + channelNonce, err := handler.GetBigInt(md, handler.PaymentChannelNonceHeader) + if err != nil { + return + } + + amount, err := handler.GetBigInt(md, handler.PaymentChannelAmountHeader) + if err != nil { + return + } + + signature, err := handler.GetBytes(md, handler.PaymentChannelSignatureHeader) + if err != nil { + return + } + + return &Payment{ + MpeContractAddress: t.mpeContractAddress(), + ChannelID: channelID, + ChannelNonce: channelNonce, + Amount: amount, + Signature: signature, + }, nil +} + +// NewTrainPaymentHandler returns new MultiPartyEscrow contract payment handler. +func NewTrainUnaryPaymentHandler( + service PaymentChannelService, + processor *blockchain.Processor, + incomeValidator IncomeUnaryValidator) handler.UnaryPaymentHandler { + return &trainUnaryPaymentHandler{ + service: service, + mpeContractAddress: processor.EscrowContractAddress, + incomeValidator: incomeValidator, + } +} + +// NewTrainPaymentHandler returns new MultiPartyEscrow contract payment handler. +func NewTrainStreamPaymentHandler( + service PaymentChannelService, + processor *blockchain.Processor, + incomeValidator IncomeStreamValidator) handler.StreamPaymentHandler { + return &trainStreamPaymentHandler{ + service: service, + mpeContractAddress: processor.EscrowContractAddress, + incomeValidator: incomeValidator, + } +} + +func (h *trainUnaryPaymentHandler) Type() (typ string) { + return TrainPaymentType +} + +func (h *trainUnaryPaymentHandler) Payment(context *handler.GrpcUnaryContext) (payment handler.Payment, err *handler.GrpcError) { + internalPayment, err := h.getPaymentFromContext(context.MD) + if err != nil { + return + } + + transaction, e := h.service.StartPaymentTransaction(internalPayment) + if e != nil { + return nil, paymentErrorToGrpcError(e) + } + + income := big.NewInt(0) + income.Sub(internalPayment.Amount, transaction.Channel().AuthorizedAmount) + e = h.incomeValidator.Validate(&IncomeUnaryData{Income: income, GrpcContext: context}) + if e != nil { + //Make sure the transaction is Rolled back , else this will cause a lock on the channel + transaction.Rollback() + return nil, paymentErrorToGrpcError(e) + } + + return transaction, nil +} + +func (h *trainUnaryPaymentHandler) getPaymentFromContext(md metadata.MD) (payment *Payment, err *handler.GrpcError) { + channelID, err := handler.GetBigInt(md, handler.PaymentChannelIDHeader) + if err != nil { + return + } + + channelNonce, err := handler.GetBigInt(md, handler.PaymentChannelNonceHeader) + if err != nil { + return + } + + amount, err := handler.GetBigInt(md, handler.PaymentChannelAmountHeader) + if err != nil { + return + } + + signature, err := handler.GetBytes(md, handler.PaymentChannelSignatureHeader) + if err != nil { + return + } + + return &Payment{ + MpeContractAddress: h.mpeContractAddress(), + ChannelID: channelID, + ChannelNonce: channelNonce, + Amount: amount, + Signature: signature, + }, nil +} + +func (h *trainUnaryPaymentHandler) Complete(payment handler.Payment) (err *handler.GrpcError) { + if err = paymentErrorToGrpcError(payment.(*paymentTransaction).Commit()); err == nil { + PublishChannelStats(payment) + } + return err +} + +func (h *trainUnaryPaymentHandler) CompleteAfterError(payment handler.Payment, result error) (err *handler.GrpcError) { + return paymentErrorToGrpcError(payment.(*paymentTransaction).Rollback()) +} diff --git a/go.mod b/go.mod index 97566218..30436135 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/improbable-eng/grpc-web v0.15.0 - github.com/ipfs/go-cid v0.4.1 + github.com/ipfs/go-cid v0.5.0 github.com/ipfs/kubo v0.32.1 github.com/magiconair/properties v1.8.9 github.com/pkg/errors v0.9.1 @@ -30,11 +30,11 @@ require ( go.etcd.io/etcd/client/v3 v3.5.17 go.etcd.io/etcd/server/v3 v3.5.17 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.31.0 - golang.org/x/net v0.33.0 - golang.org/x/time v0.8.0 - google.golang.org/grpc v1.69.2 - google.golang.org/protobuf v1.36.1 + golang.org/x/crypto v0.32.0 + golang.org/x/net v0.34.0 + golang.org/x/time v0.9.0 + google.golang.org/grpc v1.69.4 + google.golang.org/protobuf v1.36.3 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -253,7 +253,7 @@ require ( golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.26.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/go.sum b/go.sum index f634adf4..4b2ff76a 100644 --- a/go.sum +++ b/go.sum @@ -459,6 +459,8 @@ github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= +github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= @@ -1175,6 +1177,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1263,6 +1267,8 @@ golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1348,6 +1354,8 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -1374,6 +1382,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1495,6 +1505,8 @@ google.golang.org/grpc v1.69.0 h1:quSiOM1GJPmPH5XtU+BCoVXcDVJJAzNcoyfC2cCjGkI= google.golang.org/grpc v1.69.0/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1513,6 +1525,10 @@ google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNer google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/handler/interceptors.go b/handler/stream_interceptor.go similarity index 93% rename from handler/interceptors.go rename to handler/stream_interceptor.go index 650a52a2..450c3179 100644 --- a/handler/interceptors.go +++ b/handler/stream_interceptor.go @@ -2,7 +2,6 @@ package handler import ( "fmt" - "github.com/ethereum/go-ethereum/common" "github.com/singnet/snet-daemon/v5/blockchain" "github.com/singnet/snet-daemon/v5/config" @@ -72,6 +71,8 @@ const ( PrePaidAuthTokenHeader = "snet-prepaid-auth-token-bin" DynamicPriceDerived = "snet-derived-dynamic-price-cost" + + TrainingModelId = "snet-train-model-id" ) // GrpcStreamContext contains information about gRPC call which is used to @@ -134,10 +135,10 @@ func NewGrpcErrorf(code codes.Code, format string, args ...any) *GrpcError { } } -// PaymentHandler interface which is used by gRPC interceptor to get, validate +// StreamPaymentHandler interface which is used by gRPC interceptor to get, validate // and complete payment. There are two payment handler implementations so far: // jobPaymentHandler and escrowPaymentHandler. jobPaymentHandler is deprecated. -type PaymentHandler interface { +type StreamPaymentHandler interface { // Type is a content of PaymentTypeHeader field which triggers usage of the // payment handler. Type() (typ string) @@ -227,10 +228,10 @@ func (interceptor *rateLimitInterceptor) intercept(srv any, ss grpc.ServerStream // GrpcStreamInterceptor returns gRPC interceptor to validate payment. If // blockchain is disabled then noOpInterceptor is returned. -func GrpcPaymentValidationInterceptor(serviceData *blockchain.ServiceMetadata, defaultPaymentHandler PaymentHandler, paymentHandler ...PaymentHandler) grpc.StreamServerInterceptor { +func GrpcPaymentValidationInterceptor(serviceData *blockchain.ServiceMetadata, defaultPaymentHandler StreamPaymentHandler, paymentHandler ...StreamPaymentHandler) grpc.StreamServerInterceptor { interceptor := &paymentValidationInterceptor{ defaultPaymentHandler: defaultPaymentHandler, - paymentHandlers: make(map[string]PaymentHandler), + paymentHandlers: make(map[string]StreamPaymentHandler), serviceMetadata: serviceData, } @@ -240,16 +241,16 @@ func GrpcPaymentValidationInterceptor(serviceData *blockchain.ServiceMetadata, d interceptor.paymentHandlers[handler.Type()] = handler zap.L().Info("Payment handler for type registered", zap.Any("paymentType", handler.Type())) } - return interceptor.intercept + return interceptor.streamIntercept } type paymentValidationInterceptor struct { serviceMetadata *blockchain.ServiceMetadata - defaultPaymentHandler PaymentHandler - paymentHandlers map[string]PaymentHandler + defaultPaymentHandler StreamPaymentHandler + paymentHandlers map[string]StreamPaymentHandler } -func (interceptor *paymentValidationInterceptor) intercept(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (e error) { +func (interceptor *paymentValidationInterceptor) streamIntercept(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (e error) { var err *GrpcError wrapperStream := ss // check we need to have dynamic pricing here @@ -260,21 +261,13 @@ func (interceptor *paymentValidationInterceptor) intercept(srv any, ss grpc.Serv return streamError } } + context, err := getGrpcContext(wrapperStream, info) if err != nil { return err.Err() } - zap.L().Debug("New gRPC call received", zap.Any("context", context)) - - if context.Info.FullMethod == "/training_daemon.Daemon/upload_and_validate" { - e = handler(srv, wrapperStream) - if e != nil { - zap.L().Warn("gRPC handler returned error", zap.Error(e)) - return e - } - return nil - } + zap.L().Debug("[streamIntercept] New gRPC call received", zap.Any("context", context)) paymentHandler, err := interceptor.getPaymentHandler(context) if err != nil { @@ -306,7 +299,7 @@ func (interceptor *paymentValidationInterceptor) intercept(srv any, ss grpc.Serv } }() - zap.L().Debug("New payment received", zap.Any("payment", payment)) + zap.L().Debug("[streamIntercept] New payment received", zap.Any("payment", payment)) e = handler(srv, wrapperStream) if e != nil { @@ -331,7 +324,7 @@ func getGrpcContext(serverStream grpc.ServerStream, info *grpc.StreamServerInfo) }, nil } -func (interceptor *paymentValidationInterceptor) getPaymentHandler(context *GrpcStreamContext) (handler PaymentHandler, err *GrpcError) { +func (interceptor *paymentValidationInterceptor) getPaymentHandler(context *GrpcStreamContext) (handler StreamPaymentHandler, err *GrpcError) { paymentTypeMd, ok := context.MD[PaymentTypeHeader] if !ok || len(paymentTypeMd) == 0 { zap.L().Debug("Payment type was not set by caller, return default payment handler", @@ -340,6 +333,7 @@ func (interceptor *paymentValidationInterceptor) getPaymentHandler(context *Grpc } paymentType := paymentTypeMd[0] + zap.L().Debug("Payment metadata", zap.String("paymentType", paymentType), zap.Any("paymentTypeMd", paymentTypeMd)) paymentHandler, ok := interceptor.paymentHandlers[paymentType] if !ok { zap.L().Error("Unexpected payment type", zap.String("paymentType", paymentType)) diff --git a/handler/unary_interceptor.go b/handler/unary_interceptor.go new file mode 100644 index 00000000..d778629d --- /dev/null +++ b/handler/unary_interceptor.go @@ -0,0 +1,147 @@ +package handler + +import ( + "context" + "fmt" + "github.com/singnet/snet-daemon/v5/blockchain" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" +) + +type GrpcUnaryContext struct { + MD metadata.MD + Info *grpc.UnaryServerInfo +} + +type UnaryPaymentHandler interface { + // Type is a content of PaymentTypeHeader field which triggers usage of the + // payment handler. + Type() (typ string) + // Payment extracts payment data from gRPC request context and checks + // validity of payment data. It returns nil if data is valid or + // appropriate gRPC status otherwise. + Payment(context *GrpcUnaryContext) (payment Payment, err *GrpcError) + // Complete completes payment if gRPC call was successfully proceeded by + // service. + Complete(payment Payment) (err *GrpcError) + // CompleteAfterError completes payment if service returns error. + CompleteAfterError(payment Payment, result error) (err *GrpcError) +} + +type paymentValidationUnaryInterceptor struct { + serviceMetadata *blockchain.ServiceMetadata + defaultPaymentHandler UnaryPaymentHandler + paymentHandlers map[string]UnaryPaymentHandler +} + +func (interceptor *paymentValidationUnaryInterceptor) unaryIntercept(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, e error) { + var err *GrpcError + + // pass non training requests and free requests + if info.FullMethod != "/training_daemon.Daemon/validate_model" && info.FullMethod != "/training_daemon.Daemon/train_model" { + resp, e := handler(ctx, req) + if e != nil { + zap.L().Warn("gRPC handler returned error", zap.Error(e)) + return resp, e + } + return resp, e + } + + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + zap.L().Error("Invalid metadata", zap.Any("info", info)) + return nil, NewGrpcError(codes.InvalidArgument, "missing metadata").Err() + } + + c := &GrpcUnaryContext{ + MD: md, + Info: info, + } + + zap.L().Debug("[unaryIntercept] grpc metadata", zap.Any("md", c.MD)) + + zap.L().Debug("[unaryIntercept] New gRPC call received", zap.Any("context", c)) + + paymentHandler, err := interceptor.getPaymentHandler(c) + if err != nil { + return nil, err.Err() + } + + payment, err := paymentHandler.Payment(c) + if err != nil { + return nil, err.Err() + } + + defer func() { + if r := recover(); r != nil { + zap.L().Warn("Service handler called panic(panicValue)", zap.Any("panicValue", r)) + paymentHandler.CompleteAfterError(payment, fmt.Errorf("service handler called panic(%v)", r)) + panic("re-panic after payment handler error handling") + } else if e == nil { + err = paymentHandler.Complete(payment) + if err != nil { + // return err.Err() + e = err.Err() + } + } else { + err = paymentHandler.CompleteAfterError(payment, e) + if err != nil { + // return err.Err() + e = err.Err() + } + } + }() + + zap.L().Debug("[unaryIntercept] New payment received", zap.Any("payment", payment)) + + resp, e = handler(ctx, req) + if e != nil { + zap.L().Warn("gRPC handler returned error", zap.Error(e)) + return resp, e + } + + return resp, e +} + +func (interceptor *paymentValidationUnaryInterceptor) getPaymentHandler(context *GrpcUnaryContext) (handler UnaryPaymentHandler, err *GrpcError) { + paymentTypeMd, ok := context.MD[PaymentTypeHeader] + if !ok || len(paymentTypeMd) == 0 { + zap.L().Debug("Payment type was not set by caller, return default payment handler", + zap.String("defaultPaymentHandlerType", interceptor.defaultPaymentHandler.Type())) + return interceptor.defaultPaymentHandler, nil + } + + paymentType := paymentTypeMd[0] + zap.L().Debug("Payment metadata", zap.String("paymentType", paymentType), zap.Any("paymentTypeMd", paymentTypeMd)) + paymentHandler, ok := interceptor.paymentHandlers[paymentType] + if !ok { + zap.L().Error("Unexpected payment type", zap.String("paymentType", paymentType)) + return nil, NewGrpcErrorf(codes.InvalidArgument, "unexpected \"%v\", value: \"%v\"", PaymentTypeHeader, paymentType) + } + + zap.L().Debug("Return payment handler by type", zap.Any("paymentType", paymentType)) + return paymentHandler, nil +} + +func GrpcPaymentValidationUnaryInterceptor(serviceData *blockchain.ServiceMetadata, defaultPaymentHandler UnaryPaymentHandler, paymentHandler ...UnaryPaymentHandler) grpc.UnaryServerInterceptor { + interceptor := &paymentValidationUnaryInterceptor{ + defaultPaymentHandler: defaultPaymentHandler, + paymentHandlers: make(map[string]UnaryPaymentHandler), + serviceMetadata: serviceData, + } + + interceptor.paymentHandlers[defaultPaymentHandler.Type()] = defaultPaymentHandler + zap.L().Info("Default payment handler registered", zap.Any("defaultPaymentType", defaultPaymentHandler.Type())) + for _, handler := range paymentHandler { + interceptor.paymentHandlers[handler.Type()] = handler + zap.L().Info("Payment handler for type registered", zap.Any("paymentType", handler.Type())) + } + return interceptor.unaryIntercept +} + +// NoOpUnaryInterceptor is a gRPC interceptor which doesn't do payment checking. +func NoOpUnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return handler(ctx, req) +} diff --git a/pricing/pricing_strategy.go b/pricing/pricing_strategy.go index ec863625..0dfa6fe8 100644 --- a/pricing/pricing_strategy.go +++ b/pricing/pricing_strategy.go @@ -19,16 +19,16 @@ type PricingStrategy struct { } // Figure out which price type is to be used -func (pricing PricingStrategy) determinePricingApplicable(context *handler.GrpcStreamContext) (priceType PriceType, err error) { +func (pricing PricingStrategy) determinePricingApplicable(fullMethod string) (priceType PriceType, err error) { //For future, there could be multiple pricingTypes to select from and this method will help decide which pricing to pick //but for now, we just have one pricing Type ( either Fixed Price or Fixed price per Method) if config.GetBool(config.EnableDynamicPricing) { //Use Dynamic pricing ONLY when you find the mapped price method to be called. - if _, ok := pricing.serviceMetaData.GetDynamicPricingMethodAssociated(context.Info.FullMethod); ok { + if _, ok := pricing.serviceMetaData.GetDynamicPricingMethodAssociated(fullMethod); ok { return pricing.pricingTypes[DYNAMIC_PRICING], nil } else { - zap.L().Info("No Dynamic Price method defined in service proto", zap.String("Method", context.Info.FullMethod)) + zap.L().Info("No Dynamic Price method defined in service proto", zap.String("Method", fullMethod)) } } //Default pricing is Fixed Pricing @@ -55,7 +55,7 @@ func (pricing *PricingStrategy) AddPricingTypes(priceType PriceType) { func (pricing PricingStrategy) GetPrice(GrpcContext *handler.GrpcStreamContext) (price *big.Int, err error) { //Based on the input request , determine which price type is to be used - if priceType, err := pricing.determinePricingApplicable(GrpcContext); err != nil { + if priceType, err := pricing.determinePricingApplicable(GrpcContext.Info.FullMethod); err != nil { return nil, err } else { return priceType.GetPrice(GrpcContext) diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index b08c4e00..d5a25eaf 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -31,15 +31,16 @@ import ( ) type Components struct { - allowedUserPaymentHandler handler.PaymentHandler + allowedUserPaymentHandler handler.StreamPaymentHandler serviceMetadata *blockchain.ServiceMetadata blockchain *blockchain.Processor etcdClient *etcddb.EtcdClient etcdServer *etcddb.EtcdServer atomicStorage storage.AtomicStorage paymentChannelService escrow.PaymentChannelService - escrowPaymentHandler handler.PaymentHandler - grpcInterceptor grpc.StreamServerInterceptor + escrowPaymentHandler handler.StreamPaymentHandler + grpcStreamInterceptor grpc.StreamServerInterceptor + grpcUnaryInterceptor grpc.UnaryServerInterceptor paymentChannelStateService *escrow.PaymentChannelStateService etcdLockerStorage *storage.PrefixedAtomicStorage mpeSpecificStorage *storage.PrefixedAtomicStorage @@ -51,10 +52,12 @@ type Components struct { configurationService *configuration_service.ConfigurationService configurationBroadcaster *configuration_service.MessageBroadcaster organizationMetaData *blockchain.OrganizationMetaData - prepaidPaymentHandler handler.PaymentHandler + prepaidPaymentHandler handler.StreamPaymentHandler prepaidUserStorage storage.TypedAtomicStorage prepaidUserService escrow.PrePaidService - freeCallPaymentHandler handler.PaymentHandler + freeCallPaymentHandler handler.StreamPaymentHandler + trainUnaryPaymentHandler handler.UnaryPaymentHandler + trainStreamPaymentHandler handler.StreamPaymentHandler freeCallUserService escrow.FreeCallUserService freeCallUserStorage *escrow.FreeCallUserStorage freeCallLockerStorage *storage.PrefixedAtomicStorage @@ -299,7 +302,7 @@ func (components *Components) FreeCallUserService() escrow.FreeCallUserService { return components.freeCallUserService } -func (components *Components) EscrowPaymentHandler() handler.PaymentHandler { +func (components *Components) EscrowPaymentHandler() handler.StreamPaymentHandler { if components.escrowPaymentHandler != nil { return components.escrowPaymentHandler } @@ -307,13 +310,41 @@ func (components *Components) EscrowPaymentHandler() handler.PaymentHandler { components.escrowPaymentHandler = escrow.NewPaymentHandler( components.PaymentChannelService(), components.Blockchain(), - escrow.NewIncomeValidator(components.PricingStrategy()), + escrow.NewIncomeStreamValidator(components.PricingStrategy(), components.ModelStorage()), ) return components.escrowPaymentHandler } -func (components *Components) FreeCallPaymentHandler() handler.PaymentHandler { +func (components *Components) TrainUnaryPaymentHandler() handler.UnaryPaymentHandler { + if components.trainUnaryPaymentHandler != nil { + return components.trainUnaryPaymentHandler + } + + components.trainUnaryPaymentHandler = escrow.NewTrainUnaryPaymentHandler( + components.PaymentChannelService(), + components.Blockchain(), + escrow.NewTrainValidator(components.ModelStorage()), + ) + + return components.trainUnaryPaymentHandler +} + +func (components *Components) TrainStreamPaymentHandler() handler.StreamPaymentHandler { + if components.trainStreamPaymentHandler != nil { + return components.trainStreamPaymentHandler + } + + components.trainStreamPaymentHandler = escrow.NewTrainStreamPaymentHandler( + components.PaymentChannelService(), + components.Blockchain(), + escrow.NewIncomeStreamValidator(components.PricingStrategy(), components.ModelStorage()), + ) + + return components.trainStreamPaymentHandler +} + +func (components *Components) FreeCallPaymentHandler() handler.StreamPaymentHandler { if components.freeCallPaymentHandler != nil { return components.freeCallPaymentHandler } @@ -324,7 +355,8 @@ func (components *Components) FreeCallPaymentHandler() handler.PaymentHandler { return components.freeCallPaymentHandler } -func (components *Components) AllowedUserPaymentHandler() handler.PaymentHandler { +// AllowedUserPaymentHandler Only for testing when blockchain disabled +func (components *Components) AllowedUserPaymentHandler() handler.StreamPaymentHandler { if components.allowedUserPaymentHandler != nil { return components.allowedUserPaymentHandler } @@ -334,9 +366,9 @@ func (components *Components) AllowedUserPaymentHandler() handler.PaymentHandler return components.allowedUserPaymentHandler } -func (components *Components) PrePaidPaymentHandler() handler.PaymentHandler { +func (components *Components) PrePaidPaymentHandler() handler.StreamPaymentHandler { if components.prepaidPaymentHandler != nil { - return components.PrePaidPaymentHandler() + return components.prepaidPaymentHandler } components.prepaidPaymentHandler = escrow. @@ -359,9 +391,9 @@ func (components *Components) PrePaidService() escrow.PrePaidService { } // Add a chain of interceptors -func (components *Components) GrpcInterceptor() grpc.StreamServerInterceptor { - if components.grpcInterceptor != nil { - return components.grpcInterceptor +func (components *Components) GrpcStreamInterceptor() grpc.StreamServerInterceptor { + if components.grpcStreamInterceptor != nil { + return components.grpcStreamInterceptor } // Metering is now mandatory in Daemon metrics.SetDaemonGrpId(components.OrganizationMetaData().GetGroupIdString()) @@ -374,14 +406,24 @@ func (components *Components) GrpcInterceptor() grpc.StreamServerInterceptor { " as part of service publication process", zap.Error(err)) } - components.grpcInterceptor = grpc_middleware.ChainStreamServer( + components.grpcStreamInterceptor = grpc_middleware.ChainStreamServer( handler.GrpcMeteringInterceptor(), handler.GrpcRateLimitInterceptor(components.ChannelBroadcast()), - components.GrpcPaymentValidationInterceptor()) + components.GrpcStreamPaymentValidationInterceptor()) } else { - components.grpcInterceptor = grpc_middleware.ChainStreamServer(handler.GrpcRateLimitInterceptor(components.ChannelBroadcast()), - components.GrpcPaymentValidationInterceptor()) + components.grpcStreamInterceptor = grpc_middleware.ChainStreamServer(handler.GrpcRateLimitInterceptor(components.ChannelBroadcast()), + components.GrpcStreamPaymentValidationInterceptor()) + } + return components.grpcStreamInterceptor +} + +func (components *Components) GrpcUnaryInterceptor() grpc.UnaryServerInterceptor { + if components.grpcUnaryInterceptor != nil { + return components.grpcUnaryInterceptor + } + if components.Blockchain().Enabled() { + components.grpcUnaryInterceptor = components.GrpcUnaryPaymentValidationInterceptor() } - return components.grpcInterceptor + return components.grpcUnaryInterceptor } // Metering end point authentication is now mandatory for daemon @@ -449,7 +491,7 @@ type VerifyMeteringResponse struct { Data string `json:"data"` } -func (components *Components) GrpcPaymentValidationInterceptor() grpc.StreamServerInterceptor { +func (components *Components) GrpcStreamPaymentValidationInterceptor() grpc.StreamServerInterceptor { if !components.Blockchain().Enabled() { if config.GetBool(config.AllowedUserFlag) { zap.L().Info("Blockchain is disabled And AllowedUserFlag is enabled") @@ -460,8 +502,17 @@ func (components *Components) GrpcPaymentValidationInterceptor() grpc.StreamServ } else { zap.L().Info("Blockchain is enabled: instantiate payment validation interceptor") return handler.GrpcPaymentValidationInterceptor(components.ServiceMetaData(), components.EscrowPaymentHandler(), - components.FreeCallPaymentHandler(), components.PrePaidPaymentHandler()) + components.FreeCallPaymentHandler(), components.PrePaidPaymentHandler(), components.TrainStreamPaymentHandler()) + } +} + +func (components *Components) GrpcUnaryPaymentValidationInterceptor() grpc.UnaryServerInterceptor { + if components.Blockchain().Enabled() { + zap.L().Info("Blockchain is enabled: instantiate payment validation interceptor") + return handler.GrpcPaymentValidationUnaryInterceptor(components.ServiceMetaData(), components.TrainUnaryPaymentHandler()) } + zap.L().Info("Blockchain is disabled: no payment validation") + return handler.NoOpUnaryInterceptor } func (components *Components) PaymentChannelStateService() (service escrow.PaymentChannelStateServiceServer) { @@ -581,7 +632,7 @@ func (components *Components) ModelStorage() *training.ModelStorage { if components.modelStorage != nil { return components.modelStorage } - components.modelStorage = training.NewModelStorage(components.AtomicStorage()) + components.modelStorage = training.NewModelStorage(components.AtomicStorage(), components.OrganizationMetaData()) return components.modelStorage } @@ -594,7 +645,7 @@ func (components *Components) TrainingService() training.DaemonServer { return components.trainingService } - components.trainingService = training.NewTrainingService(components.PaymentChannelService(), components.ServiceMetaData(), + components.trainingService = training.NewTrainingService(components.ServiceMetaData(), components.OrganizationMetaData(), components.ModelStorage(), components.ModelUserStorage(), components.PendingModelStorage()) return components.trainingService } diff --git a/snetd/cmd/serve.go b/snetd/cmd/serve.go index 7b25192d..21d5dd70 100644 --- a/snetd/cmd/serve.go +++ b/snetd/cmd/serve.go @@ -208,7 +208,8 @@ func (d *daemon) start() { maxsizeOpt := grpc.MaxRecvMsgSize(config.GetInt(config.MaxMessageSizeInMB) * 1024 * 1024) d.grpcServer = grpc.NewServer( grpc.UnknownServiceHandler(handler.NewGrpcHandler(d.components.ServiceMetaData())), - grpc.StreamInterceptor(d.components.GrpcInterceptor()), + grpc.StreamInterceptor(d.components.GrpcStreamInterceptor()), + grpc.UnaryInterceptor(d.components.GrpcUnaryInterceptor()), maxsizeOpt, ) escrow.RegisterPaymentChannelStateServiceServer(d.grpcServer, d.components.PaymentChannelStateService()) diff --git a/training/service.go b/training/service.go index cb0862a0..223371f4 100644 --- a/training/service.go +++ b/training/service.go @@ -28,7 +28,6 @@ import ( _ "embed" "github.com/singnet/snet-daemon/v5/config" - "github.com/singnet/snet-daemon/v5/escrow" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -45,7 +44,6 @@ var TrainingProtoEmbeded string type ModelService struct { serviceMetaData *blockchain.ServiceMetadata organizationMetaData *blockchain.OrganizationMetaData - channelService escrow.PaymentChannelService storage *ModelStorage userStorage *ModelUserStorage serviceUrl string @@ -54,7 +52,6 @@ type ModelService struct { type DaemonService struct { serviceMetaData *blockchain.ServiceMetadata organizationMetaData *blockchain.OrganizationMetaData - channelService escrow.PaymentChannelService storage *ModelStorage userStorage *ModelUserStorage pendingStorage *PendingModelStorage @@ -66,7 +63,6 @@ type DaemonService struct { func NewDaemonsService( serviceMetaData *blockchain.ServiceMetadata, organizationMetaData *blockchain.OrganizationMetaData, - channelService escrow.PaymentChannelService, storage *ModelStorage, userStorage *ModelUserStorage, pendingStorage *PendingModelStorage, @@ -77,7 +73,6 @@ func NewDaemonsService( return &DaemonService{ serviceMetaData: serviceMetaData, organizationMetaData: organizationMetaData, - channelService: channelService, storage: storage, userStorage: userStorage, pendingStorage: pendingStorage, @@ -184,7 +179,7 @@ func (ds *DaemonService) getPendingModelIds() (*PendingModelData, error) { } func (ds *DaemonService) startUpdateModelStatusWorker(ctx context.Context, modelId string) { - modelKey := ds.buildModelKey(modelId) + modelKey := ds.storage.buildModelKey(modelId) currentModelData, ok, err := ds.storage.Get(modelKey) if err != nil { zap.L().Error("err in getting modelData from storage", zap.Error(err)) @@ -263,19 +258,22 @@ func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, int func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { conn, client, err := ds.getServiceClient() if client == nil || err != nil { - return &PriceInBaseUnit{ - Price: 0, - }, fmt.Errorf("issue with service: %v", err) + return nil, fmt.Errorf("issue with service: %v", err) } - price, err := client.ValidateModelPrice(context.Background(), &ValidateRequest{ + price, err := client.ValidateModelPrice(ctx, &ValidateRequest{ ModelId: request.ModelId, TrainingDataLink: request.TrainingDataLink, }) closeConn(conn) - if err != nil || price == nil { - zap.L().Error("issue with ValidateModelPrice", zap.Error(err)) + if err != nil { + zap.L().Debug("[ValidateModelPrice] can't update model prices") return nil, fmt.Errorf("issue with service: %v", err) } + err = ds.updateModelPrices(request.ModelId, price, nil) + if err != nil { + zap.L().Debug("[ValidateModelPrice] can't update model prices") + return nil, err + } return price, nil } @@ -284,14 +282,14 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer var fullData bytes.Buffer var modelID string + zap.L().Debug("start upload and validate") + conn, client, err := ds.getServiceClient() if err != nil { zap.L().Debug(err.Error()) return err } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*55) - defer cancel() - grpcStream, err := client.UploadAndValidate(ctx) + grpcStream, err := client.UploadAndValidate(context.Background()) if err != nil { zap.L().Error("error in sending UploadAndValidate", zap.Error(err)) return err @@ -305,6 +303,10 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer } if err == io.EOF { zap.L().Debug("[UploadAndValidate] EOF") + err := grpcStream.CloseSend() + if err != nil { + zap.L().Error("[UploadAndValidate] CloseSend error", zap.Error(err)) + } break } if err != nil { @@ -319,7 +321,8 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer return err } - zap.L().Debug("Received chunk of data for model", zap.String("modelID", modelID)) + zap.L().Debug("Received chunk of data for model", zap.String("modelID", req.UploadInput.ModelId)) + modelID = req.UploadInput.ModelId fullData.Write(req.UploadInput.Data) } zap.L().Debug("Received file for model %s with size %d bytes", zap.String("modelID", modelID), zap.Int("len", fullData.Len())) @@ -350,17 +353,21 @@ func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidat func (ds *DaemonService) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { conn, client, err := ds.getServiceClient() if client == nil || err != nil { - return &PriceInBaseUnit{ - Price: 0, - }, fmt.Errorf("issue with service: %v", err) + return nil, fmt.Errorf("issue with service: %v", err) } price, err := client.TrainModelPrice(ctx, &ModelID{ ModelId: request.ModelId, }) closeConn(conn) if err != nil { + zap.L().Debug("[TrainModelPrice] can't update model prices") return nil, fmt.Errorf("issue with service: %v", err) } + err = ds.updateModelPrices(request.ModelId, nil, price) + if err != nil { + zap.L().Debug("can't update model prices") + return nil, err + } return price, nil } @@ -418,8 +425,8 @@ func (ds *DaemonService) UpdateModel(ctx context.Context, request *UpdateModelRe func (ds *DaemonService) GetMethodMetadata(ctx context.Context, request *MethodMetadataRequest) (*MethodMetadata, error) { if request.GetModelId() != "" { - data, err := ds.getModelData(request.ModelId) - if err != nil { + data, err := ds.storage.GetModel(request.ModelId) + if err != nil || data == nil { zap.L().Error("[GetMethodMetadata] can't get model data", zap.Error(err)) return nil, fmt.Errorf(" can't get model data: %v", err) } @@ -486,7 +493,7 @@ func (ds *DaemonService) getServiceClient() (conn *grpc.ClientConn, client Model } func (ds *DaemonService) createModelDetails(request *NewModelRequest, response *ModelID) (data *ModelData, err error) { - key := ds.buildModelKey(response.ModelId) + key := ds.storage.buildModelKey(response.ModelId) data = ds.getModelDataToCreate(request, response) //store the model details in etcd zap.L().Debug("createModelDetails", zap.Any("key", key)) @@ -563,7 +570,7 @@ func (ds *DaemonService) deleteUserModelDetails(key *ModelKey, data *ModelData) } func (ds *DaemonService) deleteModelDetails(req *CommonRequest) (data *ModelData, err error) { - key := ds.getModelKey(req.ModelId) + key := ds.storage.buildModelKey(req.ModelId) ok := false data, ok, err = ds.storage.Get(key) if data == nil || !ok || err != nil { @@ -600,7 +607,7 @@ func convertModelDataToBO(data *ModelData) (responseData *ModelResponse) { } func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data *ModelData, err error) { - key := ds.getModelKey(request.ModelId) + key := ds.storage.buildModelKey(request.ModelId) oldAddresses := make([]string, 0) var latestAddresses []string // by default add the creator to the Authorized list of Address @@ -608,7 +615,7 @@ func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data * latestAddresses = request.AddressList } latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) // add creator - if data, err = ds.getModelData(request.ModelId); err == nil && data != nil { + if data, err = ds.storage.GetModel(request.ModelId); err == nil && data != nil { oldAddresses = data.AuthorizedAddresses data.AuthorizedAddresses = latestAddresses latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) @@ -665,12 +672,12 @@ func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address return } -func (ds *DaemonService) updateModelStatus(request *CommonRequest, newStatus Status) (data *ModelData, err error) { +func (ds *DaemonService) updateModelStatus(modelID string, newStatus Status) (data *ModelData, err error) { key := &ModelKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - ModelId: request.ModelId, + ModelId: modelID, } ok := false zap.L().Debug("[updateModelStatus]", zap.String("modelID", key.ModelId)) @@ -687,37 +694,30 @@ func (ds *DaemonService) updateModelStatus(request *CommonRequest, newStatus Sta return } -func (ds *DaemonService) buildModelKey(modelID string) (key *ModelKey) { - key = &ModelKey{ +func (ds *DaemonService) updateModelPrices(modelID string, validatePrice, trainPrice *PriceInBaseUnit) error { + key := &ModelKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.Model.GrpcMethodName, - //GRPCServiceName: request.Model.GrpcServiceName, - ModelId: modelID, + ModelId: modelID, } - return -} - -func (ds *DaemonService) getModelKey(modelID string) (key *ModelKey) { - key = &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.UpdateModelDetails.GrpcMethodName, - //GRPCServiceName: request.UpdateModelDetails.GrpcServiceName, - ModelId: modelID, + zap.L().Debug("[updateModelPrices]", zap.String("modelID", key.ModelId)) + data, ok, err := ds.storage.Get(key) + if err != nil || !ok || data == nil { + zap.L().Error("[updateModelPrices] can't get model data from etcd", zap.Error(err)) + return errors.New("can't get model data from etcd") } - return -} - -func (ds *DaemonService) getModelData(modelID string) (data *ModelData, err error) { - key := ds.getModelKey(modelID) - ok := false - if data, ok, err = ds.storage.Get(key); err != nil || !ok { - zap.L().Warn("unable to retrieve model data from storage", zap.String("Model Id", key.ModelId), zap.Error(err)) + if validatePrice != nil { + data.ValidatePrice = validatePrice.Price } - return + if trainPrice != nil { + data.TrainPrice = trainPrice.Price + } + if err = ds.storage.Put(key, data); err != nil { + zap.L().Error("[updateModelPrices] issue with updating model data", zap.Error(err)) + return fmt.Errorf("[updateModelPrices] issue with updating model data: %s", err) + } + return nil } func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (response *ModelsResponse, err error) { @@ -738,9 +738,7 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.GrpcMethodName, - //GRPCServiceName: request.GrpcServiceName, - UserAddress: request.Authorization.SignerAddress, + UserAddress: request.Authorization.SignerAddress, } modelDetailsArray := make([]*ModelResponse, 0) @@ -750,9 +748,7 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - //GRPCMethodName: request.GrpcMethodName, - //GRPCServiceName: request.GrpcServiceName, - ModelId: modelId, + ModelId: modelId, } if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { boModel := convertModelDataToBO(modelData) @@ -885,15 +881,15 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking GetModelStatus, service-provider should realize it") } zap.L().Info("[GetModelStatus] response from service-provider", zap.Any("response", responseStatus)) - zap.L().Info("[GetModelStatus] updating model status based on response from UpdateModel") - data, err := ds.updateModelStatus(request, responseStatus.Status) + zap.L().Debug("[GetModelStatus] updating model status based on response from UpdateModel") + data, err := ds.updateModelStatus(request.ModelId, responseStatus.Status) closeConn(conn) zap.L().Debug("[GetModelStatus] data that be returned to client", zap.Any("data", data)) if err == nil && data != nil { response = BuildModelResponse(data, responseStatus.Status) } else { zap.L().Error("[GetModelStatus] BuildModelResponse error", zap.Error(err)) - return response, fmt.Errorf("[GetModelStatus] issue with storing Model Id in the Daemon Storage %v", err) + return response, fmt.Errorf("issue with storage %v", err) } } else { return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("[GetModelStatus] error in invoking service for Model Training") @@ -902,13 +898,14 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re } // NewTrainingService daemon self server -func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData *blockchain.ServiceMetadata, +func NewTrainingService(serMetaData *blockchain.ServiceMetadata, orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage, pendingStorage *PendingModelStorage) DaemonServer { linkerFiles := getFileDescriptors(serMetaData.ProtoFiles) serMetaData.ProtoDescriptors = linkerFiles methodsMD, trainMD, err := parseTrainingMetadata(linkerFiles) if err != nil { + zap.L().Error(err.Error()) // TODO return nil } @@ -917,14 +914,13 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData if config.IsValidUrl(serviceURL) && config.GetBool(config.BlockchainEnabledKey) { daemonService := &DaemonService{ - channelService: channelService, serviceMetaData: serMetaData, organizationMetaData: orgMetadata, storage: storage, userStorage: userStorage, pendingStorage: pendingStorage, serviceUrl: serviceURL, - trainingMetadata: &trainMD, + trainingMetadata: trainMD, methodsMetadata: methodsMD, } @@ -936,9 +932,19 @@ func NewTrainingService(channelService escrow.PaymentChannelService, serMetaData return &NoTrainingService{} } -// parseTrainingMetadata TODO add comment -func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMetadata, trainingMD TrainingMetadata, err error) { +// parseTrainingMetadata parses metadata from Protobuf files to identify training-related methods +// and their associated metadata. +// Input: +// - protos: a collection of Protobuf files containing definitions of services and methods. +// Output: +// - methodsMD: a map where the key is the combination of service and method names, +// and the value is metadata related to the method (MethodMetadata). +// - trainingMD: a structure containing metadata for training methods, including +// whether training methods are defined and their names grouped by service. +// - err: an error, if any occurred during the parsing process. +func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMetadata, trainingMD *TrainingMetadata, err error) { methodsMD = make(map[string]*MethodMetadata) + trainingMD = &TrainingMetadata{} trainingMD.TrainingMethods = make(map[string]*structpb.ListValue) for _, protoFile := range protos { @@ -1026,6 +1032,7 @@ func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMet return } +// getFileDescriptors converts text of proto file to bufbuild linker func getFileDescriptors(protoFiles map[string]string) linker.Files { protoFiles["training_v2.proto"] = TrainingProtoEmbeded accessor := protocompile.SourceAccessorFromMap(protoFiles) diff --git a/training/storage.go b/training/storage.go index ec323c2f..b2a7a045 100644 --- a/training/storage.go +++ b/training/storage.go @@ -2,6 +2,9 @@ package training import ( "fmt" + "github.com/singnet/snet-daemon/v5/blockchain" + "github.com/singnet/snet-daemon/v5/config" + "go.uber.org/zap" "reflect" "github.com/singnet/snet-daemon/v5/storage" @@ -9,7 +12,8 @@ import ( ) type ModelStorage struct { - delegate storage.TypedAtomicStorage + delegate storage.TypedAtomicStorage + organizationMetaData *blockchain.OrganizationMetaData } type ModelUserStorage struct { @@ -29,13 +33,13 @@ func NewUserModelStorage(atomicStorage storage.AtomicStorage) *ModelUserStorage return &ModelUserStorage{delegate: userModelStorage} } -func NewModelStorage(atomicStorage storage.AtomicStorage) *ModelStorage { +func NewModelStorage(atomicStorage storage.AtomicStorage, orgMetadata *blockchain.OrganizationMetaData) *ModelStorage { prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/modelStorage") modelStorage := storage.NewTypedAtomicStorageImpl( prefixedStorage, serializeModelKey, reflect.TypeOf(ModelKey{}), utils.Serialize, utils.Deserialize, reflect.TypeOf(ModelData{}), ) - return &ModelStorage{delegate: modelStorage} + return &ModelStorage{delegate: modelStorage, organizationMetaData: orgMetadata} } func NewPendingModelStorage(atomicStorage storage.AtomicStorage) *PendingModelStorage { @@ -78,6 +82,8 @@ type ModelData struct { IsDefault bool TrainingLink string UpdatedDate string + ValidatePrice uint64 + TrainPrice uint64 } func (data *ModelData) String() string { @@ -91,9 +97,7 @@ type ModelUserKey struct { OrganizationId string ServiceId string GroupId string - //GRPCMethodName string - //GRPCServiceName string - UserAddress string + UserAddress string } func (key *ModelUserKey) String() string { @@ -173,6 +177,25 @@ func (storage *ModelStorage) CompareAndSwap(key *ModelKey, prevState *ModelData, return storage.delegate.CompareAndSwap(key, prevState, newState) } +func (storage *ModelStorage) buildModelKey(modelID string) (key *ModelKey) { + key = &ModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: storage.organizationMetaData.GetGroupIdString(), + ModelId: modelID, + } + return +} + +func (storage *ModelStorage) GetModel(modelID string) (data *ModelData, err error) { + key := storage.buildModelKey(modelID) + ok := false + if data, ok, err = storage.Get(key); err != nil || !ok { + zap.L().Warn("unable to retrieve model data from storage", zap.String("Model Id", key.ModelId), zap.Error(err)) + } + return +} + func serializeModelUserKey(key any) (serialized string, err error) { modelUserKey := key.(*ModelUserKey) return modelUserKey.String(), nil diff --git a/training/tests/integration/daemon_service_test.go b/training/tests/integration/daemon_service_test.go index 58d8bfc2..9275208b 100644 --- a/training/tests/integration/daemon_service_test.go +++ b/training/tests/integration/daemon_service_test.go @@ -49,7 +49,7 @@ func (suite *DaemonServiceSuite) SetupSuite() { } suite.daemonService = training.NewDaemonsService( - serviceMetadata, orgMetadata, nil, modelStorage, userModelStorage, pendingModelStorage, "http://localhost:5001", nil, nil, + serviceMetadata, orgMetadata, modelStorage, userModelStorage, pendingModelStorage, "http://localhost:5001", nil, nil, ) address := "localhost:5001" From 0f7595a5307cbfdad409f0210f173f9d0e6e320f Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Mon, 10 Feb 2025 17:51:02 +0300 Subject: [PATCH 13/22] fixes for training, filters for get_all_models, update go.mod --- .github/workflows/build.yml | 2 +- .github/workflows/test.yml | 2 +- blockchain/serviceMetadata.go | 24 ++ escrow/free_call_state_service.go | 2 +- escrow/income.go | 6 +- escrow/payment_handler_test.go | 2 +- escrow/train_pay_handler.go | 2 + go.mod | 164 ++++---- go.sum | 386 ++++++++++------- snetd/cmd/components.go | 5 +- snetd/cmd/serve.go | 1 + training/service.go | 391 ++++++++---------- training/storage.go | 81 +++- training/tests/functional/service_test.go | 5 +- .../tests/integration/daemon_service_test.go | 0 training/tests/unit/storage_test.go | 68 ++- .../{training_v2.proto => training.proto} | 23 +- training/training_daemon.proto | 46 ++- training/util.go | 109 +++++ 19 files changed, 796 insertions(+), 523 deletions(-) delete mode 100644 training/tests/integration/daemon_service_test.go rename training/{training_v2.proto => training.proto} (89%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1ee3d5e..eb3c710b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: - name: download and install uses: actions/setup-go@v5 with: - go-version: '1.23.1' + go-version: '1.23.6' - name: install protoc (protobuf) uses: arduino/setup-protoc@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9271854b..4ddddee0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: - name: download and install uses: actions/setup-go@v5 with: - go-version: '1.23.1' + go-version: '1.23.6' - name: install protoc (protobuf) uses: arduino/setup-protoc@v3 diff --git a/blockchain/serviceMetadata.go b/blockchain/serviceMetadata.go index 14d60e8f..8517365e 100644 --- a/blockchain/serviceMetadata.go +++ b/blockchain/serviceMetadata.go @@ -1,10 +1,13 @@ package blockchain import ( + "context" "encoding/json" "fmt" + "github.com/bufbuild/protocompile" "github.com/bufbuild/protocompile/linker" "github.com/singnet/snet-daemon/v5/errs" + "maps" "math/big" "os" "slices" @@ -269,10 +272,31 @@ func ServiceMetaData() *ServiceMetadata { if err != nil { zap.L().Panic("error on determining service metadata from file"+errs.ErrDescURL(errs.InvalidMetadata), zap.Error(err)) } + + // if ModelTraining enabled in config training package will init protoDescriptors + if !config.GetBool(config.ModelTrainingEnabled) { + metadata.ProtoDescriptors = getFileDescriptors(metadata.ProtoFiles) + } + zap.L().Debug("service type: " + metadata.GetServiceType()) return metadata } +// getFileDescriptors converts text of proto files to bufbuild linker +func getFileDescriptors(protoFiles map[string]string) linker.Files { + accessor := protocompile.SourceAccessorFromMap(protoFiles) + r := protocompile.WithStandardImports(&protocompile.SourceResolver{Accessor: accessor}) + compiler := protocompile.Compiler{ + Resolver: r, + SourceInfoMode: protocompile.SourceInfoStandard, + } + fds, err := compiler.Compile(context.Background(), slices.Collect(maps.Keys(protoFiles))...) + if err != nil || fds == nil { + zap.L().Fatal("[getFileDescriptors] failed to analyze protofile"+errs.ErrDescURL(errs.InvalidProto), zap.Error(err)) + } + return fds +} + func ReadServiceMetaDataFromLocalFile(filename string) (*ServiceMetadata, error) { file, err := os.ReadFile(filename) if err != nil { diff --git a/escrow/free_call_state_service.go b/escrow/free_call_state_service.go index 7a56d720..40b66bc9 100644 --- a/escrow/free_call_state_service.go +++ b/escrow/free_call_state_service.go @@ -50,7 +50,7 @@ func (service *FreeCallStateService) GetFreeCallsAvailable(context context.Conte } func (service *BlockChainDisabledFreeCallStateService) GetFreeCallsAvailable(context.Context, *FreeCallStateRequest) (*FreeCallStateReply, error) { - return &FreeCallStateReply{UserId: "", FreeCallsAvailable: 0}, fmt.Errorf("error in determining free call state") + return &FreeCallStateReply{UserId: "", FreeCallsAvailable: 0}, fmt.Errorf("error in determining free calls because blockchain is disabled, contact service prodiver") } func (service *FreeCallStateService) verify(request *FreeCallStateRequest) (err error) { diff --git a/escrow/income.go b/escrow/income.go index b31cc79b..6e645698 100644 --- a/escrow/income.go +++ b/escrow/income.go @@ -2,6 +2,7 @@ package escrow import ( "errors" + "fmt" "github.com/singnet/snet-daemon/v5/pricing" "github.com/singnet/snet-daemon/v5/training" "go.uber.org/zap" @@ -103,12 +104,12 @@ func NewTrainValidator(storage *training.ModelStorage) (validator IncomeUnaryVal func (validator *trainUnaryValidator) Validate(data *IncomeUnaryData) (err error) { modelID, ok := data.GrpcContext.MD[handler.TrainingModelId] if !ok { - return errors.New("no training model found") + return errors.New("[trainUnaryValidator] no training model found") } model, err := validator.storage.GetModel(modelID[0]) if err != nil { - return errors.New("no training model found") + return errors.New("[trainUnaryValidator] no training model found") } price := big.NewInt(0) @@ -125,6 +126,7 @@ func (validator *trainUnaryValidator) Validate(data *IncomeUnaryData) (err error zap.L().Debug("[Validate]", zap.Uint64("price", price.Uint64())) if data.Income.Cmp(price) != 0 { + zap.L().Error(fmt.Sprintf("[Validate] income %d does not equal to price %d", data.Income, price)) err = NewPaymentError(Unauthenticated, "income %d does not equal to price %d", data.Income, price) return } diff --git a/escrow/payment_handler_test.go b/escrow/payment_handler_test.go index b78b81ef..df9b621c 100644 --- a/escrow/payment_handler_test.go +++ b/escrow/payment_handler_test.go @@ -20,7 +20,7 @@ type PaymentHandlerTestSuite struct { suite.Suite paymentChannelServiceMock PaymentChannelService - incomeValidatorMock IncomeValidator + incomeValidatorMock IncomeStreamValidator paymentHandler paymentChannelPaymentHandler } diff --git a/escrow/train_pay_handler.go b/escrow/train_pay_handler.go index 79a1cfcc..773ddf24 100644 --- a/escrow/train_pay_handler.go +++ b/escrow/train_pay_handler.go @@ -2,6 +2,7 @@ package escrow import ( "github.com/ethereum/go-ethereum/common" + "go.uber.org/zap" "google.golang.org/grpc/metadata" "math/big" @@ -133,6 +134,7 @@ func (h *trainUnaryPaymentHandler) Payment(context *handler.GrpcUnaryContext) (p } income := big.NewInt(0) + zap.L().Debug("[trainUnaryPaymentHandler.Payment]", zap.Any("Amount", internalPayment.Amount), zap.Any("AuthorizedAmount", transaction.Channel().AuthorizedAmount)) income.Sub(internalPayment.Amount, transaction.Channel().AuthorizedAmount) e = h.incomeValidator.Validate(&IncomeUnaryData{Income: income, GrpcContext: context}) if e != nil { diff --git a/go.mod b/go.mod index 30436135..43057742 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/OneOfOne/go-utils v0.0.0-20180319162427-6019ff89a94e github.com/bufbuild/protocompile v0.14.1 github.com/emicklei/proto v1.14.0 - github.com/ethereum/go-ethereum v1.14.12 + github.com/ethereum/go-ethereum v1.15.0 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gorilla/handlers v1.5.2 @@ -15,26 +15,26 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/improbable-eng/grpc-web v0.15.0 github.com/ipfs/go-cid v0.5.0 - github.com/ipfs/kubo v0.32.1 + github.com/ipfs/kubo v0.33.1 github.com/magiconair/properties v1.8.9 github.com/pkg/errors v0.9.1 github.com/rs/cors v1.11.1 github.com/rs/xid v1.6.0 - github.com/singnet/snet-ecosystem-contracts v0.1.1 + github.com/singnet/snet-ecosystem-contracts v0.2.1 github.com/soheilhy/cmux v0.1.5 github.com/spf13/cast v1.7.1 github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - go.etcd.io/etcd/client/v3 v3.5.17 - go.etcd.io/etcd/server/v3 v3.5.17 + go.etcd.io/etcd/client/v3 v3.5.18 + go.etcd.io/etcd/server/v3 v3.5.18 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.32.0 + golang.org/x/crypto v0.33.0 golang.org/x/net v0.34.0 - golang.org/x/time v0.9.0 - google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.36.3 + golang.org/x/time v0.10.0 + google.golang.org/grpc v1.70.0 + google.golang.org/protobuf v1.36.5 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -46,20 +46,20 @@ require ( github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.19.1 // indirect + github.com/bits-and-blooms/bitset v1.20.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/caddyserver/certmagic v0.21.4 // indirect + github.com/caddyserver/certmagic v0.21.7 // indirect github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble v1.1.2 // indirect + github.com/cockroachdb/pebble v1.1.4 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/consensys/bavard v0.1.24 // indirect - github.com/consensys/gnark-crypto v0.14.0 // indirect + github.com/consensys/bavard v0.1.29 // indirect + github.com/consensys/gnark-crypto v0.16.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect @@ -78,7 +78,8 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -87,15 +88,16 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.1 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/btree v1.1.2 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect + github.com/google/pprof v0.0.0-20250208200701-d0013a598941 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-bexpr v0.1.10 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -108,13 +110,13 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.24.3 // indirect + github.com/ipfs/boxo v0.27.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ds-measure v0.2.0 // indirect github.com/ipfs/go-fs-lock v0.0.7 // indirect - github.com/ipfs/go-ipfs-cmds v0.14.0 // indirect + github.com/ipfs/go-ipfs-cmds v0.14.1 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-cbor v0.2.0 // indirect github.com/ipfs/go-ipld-format v0.6.0 // indirect @@ -126,35 +128,36 @@ require ( github.com/ipld/go-car/v2 v2.14.2 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect - github.com/ipshipyard/p2p-forge v0.0.2 // indirect + github.com/ipshipyard/p2p-forge v0.3.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/jonboulle/clockwork v0.4.0 // indirect + github.com/jonboulle/clockwork v0.5.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/koron/go-ssdp v0.0.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/koron/go-ssdp v0.0.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect - github.com/libp2p/go-libp2p v0.37.0 // indirect + github.com/libp2p/go-libp2p v0.39.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.28.1 // indirect - github.com/libp2p/go-libp2p-kbucket v0.6.4 // indirect - github.com/libp2p/go-libp2p-record v0.2.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.29.0 // indirect + github.com/libp2p/go-libp2p-kbucket v0.6.5 // indirect + github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.4 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect - github.com/libp2p/go-netroute v0.2.1 // indirect + github.com/libp2p/go-netroute v0.2.2 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mholt/acmez/v2 v2.0.3 // indirect - github.com/miekg/dns v1.1.62 // indirect + github.com/mholt/acmez/v3 v3.0.1 // indirect + github.com/miekg/dns v1.1.63 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect @@ -164,57 +167,66 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.13.0 // indirect - github.com/multiformats/go-multiaddr-dns v0.4.0 // indirect + github.com/multiformats/go-multiaddr v0.14.0 // indirect + github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect - github.com/multiformats/go-multistream v0.5.0 // indirect + github.com/multiformats/go-multistream v0.6.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onsi/ginkgo/v2 v2.20.2 // indirect + github.com/onsi/ginkgo/v2 v2.22.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect - github.com/pion/datachannel v1.5.9 // indirect + github.com/pion/datachannel v1.5.10 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect - github.com/pion/ice/v2 v2.3.36 // indirect + github.com/pion/dtls/v3 v3.0.4 // indirect + github.com/pion/ice/v2 v2.3.37 // indirect + github.com/pion/ice/v4 v4.0.6 // indirect github.com/pion/interceptor v0.1.37 // indirect - github.com/pion/logging v0.2.2 // indirect + github.com/pion/logging v0.2.3 // indirect github.com/pion/mdns v0.0.12 // indirect + github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.14 // indirect - github.com/pion/rtp v1.8.9 // indirect - github.com/pion/sctp v1.8.33 // indirect - github.com/pion/sdp/v3 v3.0.9 // indirect + github.com/pion/rtcp v1.2.15 // indirect + github.com/pion/rtp v1.8.11 // indirect + github.com/pion/sctp v1.8.35 // indirect + github.com/pion/sdp/v3 v3.0.10 // indirect github.com/pion/srtp/v2 v2.0.20 // indirect + github.com/pion/srtp/v3 v3.0.4 // indirect github.com/pion/stun v0.6.1 // indirect + github.com/pion/stun/v2 v2.0.0 // indirect + github.com/pion/stun/v3 v3.0.0 // indirect github.com/pion/transport/v2 v2.2.10 // indirect + github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/turn/v2 v2.1.6 // indirect - github.com/pion/webrtc/v3 v3.3.4 // indirect + github.com/pion/turn/v4 v4.0.0 // indirect + github.com/pion/webrtc/v3 v3.3.5 // indirect + github.com/pion/webrtc/v4 v4.0.8 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.60.0 // indirect + github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.48.1 // indirect + github.com/quic-go/quic-go v0.49.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/samber/lo v1.47.0 // indirect + github.com/samber/lo v1.49.1 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/afero v1.12.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/supranational/blst v0.3.13 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect @@ -224,43 +236,45 @@ require ( github.com/urfave/cli/v2 v2.25.7 // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect - github.com/whyrusleeping/cbor-gen v0.1.2 // indirect + github.com/whyrusleeping/cbor-gen v0.2.0 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/blake3 v0.2.4 // indirect - go.etcd.io/bbolt v1.3.11 // indirect - go.etcd.io/etcd/api/v3 v3.5.17 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.17 // indirect - go.etcd.io/etcd/client/v2 v2.305.17 // indirect - go.etcd.io/etcd/pkg/v3 v3.5.17 // indirect - go.etcd.io/etcd/raft/v3 v3.5.17 // indirect + go.etcd.io/bbolt v1.4.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.18 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.18 // indirect + go.etcd.io/etcd/client/v2 v2.305.18 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.18 // indirect + go.etcd.io/etcd/raft/v3 v3.5.18 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect - go.opentelemetry.io/otel/sdk v1.31.0 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/dig v1.18.0 // indirect go.uber.org/fx v1.23.0 // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/tools v0.26.0 // indirect + golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 // indirect + golang.org/x/mod v0.23.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/tools v0.29.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - gonum.org/v1/gonum v0.15.0 // indirect - google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb // indirect + gonum.org/v1/gonum v0.15.1 // indirect + google.golang.org/genproto v0.0.0-20250207221924-e9438ea467c6 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect diff --git a/go.sum b/go.sum index 4b2ff76a..049a9d9b 100644 --- a/go.sum +++ b/go.sum @@ -77,17 +77,22 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.19.1 h1:mv2yVhy96D2CuskLPXnc58oJNMs5PCWjAZuyYU0p12M= github.com/bits-and-blooms/bitset v1.19.1/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= +github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/caddyserver/certmagic v0.21.4 h1:e7VobB8rffHv8ZZpSiZtEwnLDHUwLVYLWzWSa1FfKI0= -github.com/caddyserver/certmagic v0.21.4/go.mod h1:swUXjQ1T9ZtMv95qj7/InJvWLXURU85r+CfG0T+ZbDE= +github.com/caddyserver/certmagic v0.21.6 h1:1th6GfprVfsAtFNOu4StNMF5IxK5XiaI0yZhAHlZFPE= +github.com/caddyserver/certmagic v0.21.6/go.mod h1:n1sCo7zV1Ez2j+89wrzDxo4N/T1Ws/Vx8u5NvuBFabw= +github.com/caddyserver/certmagic v0.21.7 h1:66KJioPFJwttL43KYSWk7ErSmE6LfaJgCQuhm8Sg6fg= +github.com/caddyserver/certmagic v0.21.7/go.mod h1:LCPG3WLxcnjVKl/xpjzM0gqh0knrKKKiO5WVttX2eEI= github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -97,6 +102,7 @@ github.com/ceramicnetwork/go-dag-jose v0.1.1 h1:7pObs22egc14vSS3AfCFfS1VmaL4lQUs github.com/ceramicnetwork/go-dag-jose v0.1.1/go.mod h1:8ptnYwY2Z2y/s5oJnNBn/UCxLg6CpramNJ2ZXF/5aNY= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -116,8 +122,8 @@ github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/e github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= -github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.4 h1:5II1uEP4MyHLDnsrbv/EZ36arcb9Mxg3n+owhZ3GrG8= +github.com/cockroachdb/pebble v1.1.4/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= @@ -125,8 +131,12 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1: github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/consensys/bavard v0.1.24 h1:Lfe+bjYbpaoT7K5JTFoMi5wo9V4REGLvQQbHmatoN2I= github.com/consensys/bavard v0.1.24/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= +github.com/consensys/bavard v0.1.29 h1:fobxIYksIQ+ZSrTJUuQgu+HIJwclrAPcdXqd7H2hh1k= +github.com/consensys/bavard v0.1.29/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= +github.com/consensys/gnark-crypto v0.16.0 h1:8Dl4eYmUWK9WmlP1Bj6je688gBRJCJbT8Mw4KoTAawo= +github.com/consensys/gnark-crypto v0.16.0/go.mod h1:Ke3j06ndtPTVvo++PhGNgvm+lgpLvzbcE2MqljY7diU= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -181,8 +191,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/emicklei/proto v1.13.3 h1:h2u3JgIKSBWHI0jTFFAxiIzOQAe+bRGlQ/WlaIa49Uk= -github.com/emicklei/proto v1.13.3/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/emicklei/proto v1.14.0 h1:WYxC0OrBuuC+FUCTZvb8+fzEHdZMwLEF+OnVfZA3LXU= github.com/emicklei/proto v1.14.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= @@ -192,8 +200,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy4WDs= github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.14.12 h1:8hl57x77HSUo+cXExrURjU/w1VhL+ShCTJrTwcCQSe4= -github.com/ethereum/go-ethereum v1.14.12/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY= +github.com/ethereum/go-ethereum v1.15.0 h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv09/1ME4= +github.com/ethereum/go-ethereum v1.15.0/go.mod h1:4q+4t48P2C03sjqGvTXix5lEOplf5dz4CTosbjt5tGs= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= @@ -201,6 +209,8 @@ github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:Jp github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= +github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= @@ -215,8 +225,14 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= +github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o= +github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -262,6 +278,7 @@ github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -291,14 +308,14 @@ github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= -github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM= +github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -327,8 +344,8 @@ github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXi github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -351,8 +368,10 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20241017200806-017d972448fc h1:NGyrhhFhwvRAZg02jnYVg3GBQy0qGBKmFQJwaPmpmxs= -github.com/google/pprof v0.0.0-20241017200806-017d972448fc/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc= +github.com/google/pprof v0.0.0-20250208200701-d0013a598941/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -391,8 +410,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -443,22 +464,20 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/ipfs-shipyard/nopfs v0.0.12 h1:mvwaoefDF5VI9jyvgWCmaoTJIJFAfrbyQV5fJz35hlk= -github.com/ipfs-shipyard/nopfs v0.0.12/go.mod h1:mQyd0BElYI2gB/kq/Oue97obP4B3os4eBmgfPZ+hnrE= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.14 h1:HFepJt/MxhZ3/GsLZkkAPzIPdNYKaLO1Qb7YmPbWIKk= +github.com/ipfs-shipyard/nopfs v0.0.14/go.mod h1:mQyd0BElYI2gB/kq/Oue97obP4B3os4eBmgfPZ+hnrE= +github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcdHUd7SDsUOY= +github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.3 h1:gldDPOWdM3Rz0v5LkVLtZu7A7gFNvAlWcmxhCqlHR3c= -github.com/ipfs/boxo v0.24.3/go.mod h1:h0DRzOY1IBFDHp6KNvrJLMFdSXTYID0Zf+q7X05JsNg= +github.com/ipfs/boxo v0.27.4 h1:6nC8lY5GnR6whAbW88hFz6L13wZUj2vr5BRe3iTvYBI= +github.com/ipfs/boxo v0.27.4/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8= github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= -github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= -github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= @@ -476,14 +495,14 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ= github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE= -github.com/ipfs/go-ds-pebble v0.4.0 h1:88lgFAs2ck8jCQ8lMYRBtksEg18r9BlvTxIMnNJkZaQ= -github.com/ipfs/go-ds-pebble v0.4.0/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM= +github.com/ipfs/go-ds-pebble v0.4.2 h1:6FfU9yKpz+lTyDLwul8Oh+mEyLUQ7FWx5I82H5NSTm4= +github.com/ipfs/go-ds-pebble v0.4.2/go.mod h1:JDK6dqKXyB45MgfTsaXKWBHqc9/J4OVsvhm1juEwug0= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= -github.com/ipfs/go-ipfs-cmds v0.14.0 h1:sxdurhAHSdQr5VrSNJjc+t92uJObSNq+gRVm/wLZGMM= -github.com/ipfs/go-ipfs-cmds v0.14.0/go.mod h1:zj2jN7bHJ4pDucRmqdq863AQYcsqdxXrfVkr9eqPfvo= +github.com/ipfs/go-ipfs-cmds v0.14.1 h1:TA8vBixPwXL3k7VtcbX3r4FQgw2m+jMOWlslUOlM9Rs= +github.com/ipfs/go-ipfs-cmds v0.14.1/go.mod h1:SCYxNUVPeVR05cE8DJ6wyH2+aQ8vPgjxxkxQWOXobzo= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -516,16 +535,16 @@ github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jq github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= -github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= -github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipfs/go-peertaskqueue v0.8.2 h1:PaHFRaVFdxQk1Qo3OKiHPYjmmusQy7gKQUaL8JDszAU= +github.com/ipfs/go-peertaskqueue v0.8.2/go.mod h1:L6QPvou0346c2qPJNiJa6BvOibxDfaiPlqHInmzg0FA= github.com/ipfs/go-test v0.0.4 h1:DKT66T6GBB6PsDFLoO56QZPrOmzJkqU1FZH5C9ySkew= github.com/ipfs/go-test v0.0.4/go.mod h1:qhIM1EluEfElKKM6fnWxGn822/z9knUGM1+I/OAQNKI= github.com/ipfs/go-unixfsnode v1.9.2 h1:0A12BYs4XOtDPJTMlwmNPlllDfqcc4yie4e919hcUXk= github.com/ipfs/go-unixfsnode v1.9.2/go.mod h1:v1nuMFHf4QTIhFUdPMvg1nQu7AqDLvIdwyvJ531Ot1U= github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= -github.com/ipfs/kubo v0.32.1 h1:nkx5qrkMeJ2f1ET7v3vx7U1ycurM0dC9R7AnsuSrNjk= -github.com/ipfs/kubo v0.32.1/go.mod h1:7fi1IMPgW5fupyMFUjJ4d4zbvkTEwq6tV3T+EQvtF28= +github.com/ipfs/kubo v0.33.1 h1:dvc1o7j//9HAKjZg1d1ODSPVxmUn5YR7Ut5JPnR+H+Q= +github.com/ipfs/kubo v0.33.1/go.mod h1:qc3gVRqjx5y9mPvsEz+CO5nc1mXvy4r9Etvqga2lnMo= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.14.2 h1:9ERr7KXpCC7If0rChZLhYDlyr6Bes6yRKPJnCO3hdHY= @@ -536,8 +555,10 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= -github.com/ipshipyard/p2p-forge v0.0.2 h1:86y9LxGB8sGxYQ/If5sNx+c8C/huSpBUg3UZ1uvtym8= -github.com/ipshipyard/p2p-forge v0.0.2/go.mod h1:taPeh3PDSO8Ual0/N2tIOAUXPV8gZoPF3uPXoUyiq14= +github.com/ipshipyard/p2p-forge v0.3.0 h1:mdeFqiq8ljX149OCQvveV0vOlKeIt4PWkJjXVfux/GE= +github.com/ipshipyard/p2p-forge v0.3.0/go.mod h1:L0TJMzniMEDjX8G+RB201U2woHvASwbsujNVDNVivDo= +github.com/ipshipyard/p2p-forge v0.3.1 h1:Vr0l6wzX4zL7l8+UHJlsRBNHmlHpP3c//NrCZeGj4KU= +github.com/ipshipyard/p2p-forge v0.3.1/go.mod h1:XQAvFJeXGo4oiyVPXkC3cph//5kF785L5Pjd3/kWFWo= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -548,8 +569,8 @@ github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -571,12 +592,14 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= +github.com/koron/go-ssdp v0.0.5 h1:E1iSMxIs4WqxTbIBLtmNBeOOC+1sCIXQeqTWVnpmwhk= +github.com/koron/go-ssdp v0.0.5/go.mod h1:Qm59B7hpKpDqfyRNWRNr00jGwLdXjDyZh6y7rH6VS0w= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -601,24 +624,32 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= -github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= -github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= +github.com/libp2p/go-doh-resolver v0.5.0 h1:4h7plVVW+XTS+oUBw2+8KfoM1jF6w8XmO7+skhePFdE= +github.com/libp2p/go-doh-resolver v0.5.0/go.mod h1:aPDxfiD2hNURgd13+hfo29z9IC22fv30ee5iM31RzxU= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.37.0 h1:8K3mcZgwTldydMCNOiNi/ZJrOB9BY+GlI3UxYzxBi9A= -github.com/libp2p/go-libp2p v0.37.0/go.mod h1:GOKmSN99scDuYGTwaTbQPR8Nt6dxrK3ue7OjW2NGDg4= +github.com/libp2p/go-libp2p v0.38.2 h1:9SZQDOCi82A25An4kx30lEtr6kGTxrtoaDkbs5xrK5k= +github.com/libp2p/go-libp2p v0.38.2/go.mod h1:QWV4zGL3O9nXKdHirIC59DoRcZ446dfkjbOJ55NEWFo= +github.com/libp2p/go-libp2p v0.39.0 h1:LmrhDRud4eDkQCSB4l5NfoIFDqvDwAyANmfeYkgnKgs= +github.com/libp2p/go-libp2p v0.39.0/go.mod h1:3zicI8Lp7Isun+Afo/JOACUbbJqqR2owK6RQWFsVAbI= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= -github.com/libp2p/go-libp2p-kad-dht v0.28.1 h1:DVTfzG8Ybn88g9RycIq47evWCRss5f0Wm8iWtpwyHso= -github.com/libp2p/go-libp2p-kad-dht v0.28.1/go.mod h1:0wHURlSFdAC42+wF7GEmpLoARw8JuS8do2guCtc/Y/w= +github.com/libp2p/go-libp2p-kad-dht v0.28.2 h1:/VivUl/Ru0tVgkWNhDDBy8pK6q+gRdI+z8VfqmSUJWo= +github.com/libp2p/go-libp2p-kad-dht v0.28.2/go.mod h1:sUR/qh4p/5+YFXBtwOiCmIBeBA2YD94ttmL+Xk8+pTE= +github.com/libp2p/go-libp2p-kad-dht v0.29.0 h1:045eW21lGlMSD9aKSZZGH4fnBMIInPwQLxIQ35P962I= +github.com/libp2p/go-libp2p-kad-dht v0.29.0/go.mod h1:mIci3rHSwDsxQWcCjfmxD8vMTgh5xLuvwb1D5WP8ZNk= github.com/libp2p/go-libp2p-kbucket v0.6.4 h1:OjfiYxU42TKQSB8t8WYd8MKhYhMJeO2If+NiuKfb6iQ= github.com/libp2p/go-libp2p-kbucket v0.6.4/go.mod h1:jp6w82sczYaBsAypt5ayACcRJi0lgsba7o4TzJKEfWA= +github.com/libp2p/go-libp2p-kbucket v0.6.5 h1:Fsl1YvZcMwqrR4DYrTO02yo9PGYs2HBQIT3lGXFMTxg= +github.com/libp2p/go-libp2p-kbucket v0.6.5/go.mod h1:U6WOd0BvnSp03IQSrjgM54tg7zh1UUNsXLJqAQzClTA= github.com/libp2p/go-libp2p-pubsub v0.12.0 h1:PENNZjSfk8KYxANRlpipdS7+BfLmOl3L2E/6vSNjbdI= github.com/libp2p/go-libp2p-pubsub v0.12.0/go.mod h1:Oi0zw9aw8/Y5GC99zt+Ef2gYAl+0nZlwdJonDyOz/sE= github.com/libp2p/go-libp2p-pubsub-router v0.6.0 h1:D30iKdlqDt5ZmLEYhHELCMRj8b4sFAqrUcshIUvVP/s= github.com/libp2p/go-libp2p-pubsub-router v0.6.0/go.mod h1:FY/q0/RBTKsLA7l4vqC2cbRbOvyDotg8PJQ7j8FDudE= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= +github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.4 h1:6LqS1Bzn5CfDJ4tzvP9uwh42IB7TJLNFJA6dEeGBv84= github.com/libp2p/go-libp2p-routing-helpers v0.7.4/go.mod h1:we5WDj9tbolBXOuF1hGOkR+r7Uh1408tQbAKaT5n1LE= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= @@ -629,12 +660,13 @@ github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0 github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= -github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= -github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= +github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFPuZ8= +github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= +github.com/libp2p/go-yamux/v4 v4.0.2 h1:nrLh89LN/LEiqcFiqdKDRHjGstN300C1269K/EX0CPU= github.com/libp2p/zeroconf/v2 v2.2.0 h1:Cup06Jv6u81HLhIj1KasuNM/RHHrJ8T7wOTS4+Tv53Q= github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -662,12 +694,16 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mholt/acmez/v2 v2.0.3 h1:CgDBlEwg3QBp6s45tPQmFIBrkRIkBT4rW4orMM6p4sw= -github.com/mholt/acmez/v2 v2.0.3/go.mod h1:pQ1ysaDeGrIMvJ9dfJMk5kJNkn7L2sb3UhyrX6Q91cw= +github.com/mholt/acmez/v3 v3.0.0 h1:r1NcjuWR0VaKP2BTjDK9LRFBw/WvURx3jlaEUl9Ht8E= +github.com/mholt/acmez/v3 v3.0.0/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= +github.com/mholt/acmez/v3 v3.0.1 h1:4PcjKjaySlgXK857aTfDuRbmnM5gb3Ruz3tvoSJAUp8= +github.com/mholt/acmez/v3 v3.0.1/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= +github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= +github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= @@ -707,10 +743,10 @@ github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYg github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= -github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= -github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= -github.com/multiformats/go-multiaddr-dns v0.4.0 h1:P76EJ3qzBXpUXZ3twdCDx/kvagMsNo0LMFXpyms/zgU= -github.com/multiformats/go-multiaddr-dns v0.4.0/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= +github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= +github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= +github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= @@ -721,8 +757,8 @@ github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= -github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= +github.com/multiformats/go-multistream v0.6.0 h1:ZaHKbsL404720283o4c/IHQXiS6gb8qAN5EIJ4PN5EA= +github.com/multiformats/go-multistream v0.6.0/go.mod h1:MOyoG5otO24cHIg8kf9QW2/NozURlkP/rvi2FQJyCPg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= @@ -757,15 +793,18 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= -github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -789,6 +828,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= @@ -796,35 +837,53 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pion/datachannel v1.5.9 h1:LpIWAOYPyDrXtU+BW7X0Yt/vGtYxtXQ8ql7dFfYUVZA= -github.com/pion/datachannel v1.5.9/go.mod h1:kDUuk4CU4Uxp82NH4LQZbISULkX/HtzKa4P7ldf9izE= +github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= +github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.36 h1:SopeXiVbbcooUg2EIR8sq4b13RQ8gzrkkldOVg+bBsc= -github.com/pion/ice/v2 v2.3.36/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= +github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U= +github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg= +github.com/pion/ice/v2 v2.3.37 h1:ObIdaNDu1rCo7hObhs34YSBcO7fjslJMZV0ux+uZWh0= +github.com/pion/ice/v2 v2.3.37/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= +github.com/pion/ice/v4 v4.0.6 h1:jmM9HwI9lfetQV/39uD0nY4y++XZNPhvzIPCb8EwxUM= +github.com/pion/ice/v4 v4.0.6/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI= github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= +github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= +github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= +github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= -github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= -github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= +github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk= -github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/sctp v1.8.33 h1:dSE4wX6uTJBcNm8+YlMg7lw1wqyKHggsP5uKbdj+NZw= -github.com/pion/sctp v1.8.33/go.mod h1:beTnqSzewI53KWoG3nqB282oDMGrhNxBdb+JZnkCwRM= +github.com/pion/rtp v1.8.10 h1:puphjdbjPB+L+NFaVuZ5h6bt1g5q4kFIoI+r5q/g0CU= +github.com/pion/rtp v1.8.10/go.mod h1:8uMBJj32Pa1wwx8Fuv/AsFhn8jsgw+3rUC2PfoBZ8p4= +github.com/pion/rtp v1.8.11 h1:17xjnY5WO5hgO6SD3/NTIUPvSFw/PbLsIJyz1r1yNIk= +github.com/pion/rtp v1.8.11/go.mod h1:8uMBJj32Pa1wwx8Fuv/AsFhn8jsgw+3rUC2PfoBZ8p4= +github.com/pion/sctp v1.8.35 h1:qwtKvNK1Wc5tHMIYgTDJhfZk7vATGVHhXbUDfHbYwzA= +github.com/pion/sctp v1.8.35/go.mod h1:EcXP8zCYVTRy3W9xtOF7wJm1L1aXfKRQzaM33SjQlzg= github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/sdp/v3 v3.0.10 h1:6MChLE/1xYB+CjumMw+gZ9ufp2DPApuVSnDT8t5MIgA= +github.com/pion/sdp/v3 v3.0.10/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= github.com/pion/srtp/v2 v2.0.20 h1:HNNny4s+OUmG280ETrCdgFndp4ufx3/uy85EawYEhTk= github.com/pion/srtp/v2 v2.0.20/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= +github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M= +github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= +github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= +github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= +github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= @@ -836,8 +895,12 @@ github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uP github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.3.4 h1:v2heQVnXTSqNRXcaFQVOhIOYkLMxOu1iJG8uy1djvkk= -github.com/pion/webrtc/v3 v3.3.4/go.mod h1:liNa+E1iwyzyXqNUwvoMRNQ10x8h8FOeJKL8RkIbamE= +github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM= +github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA= +github.com/pion/webrtc/v3 v3.3.5 h1:ZsSzaMz/i9nblPdiAkZoP+E6Kmjw+jnyq3bEmU3EtRg= +github.com/pion/webrtc/v3 v3.3.5/go.mod h1:liNa+E1iwyzyXqNUwvoMRNQ10x8h8FOeJKL8RkIbamE= +github.com/pion/webrtc/v4 v4.0.8 h1:T1ZmnT9qxIJIt4d8XoiMOBrTClGHDDXNg9e/fh018Qc= +github.com/pion/webrtc/v4 v4.0.8/go.mod h1:HHBeUVBAC+j4ZFnYhovEFStF02Arb1EyD4G7e7HBTJw= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -872,8 +935,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA= -github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -885,8 +948,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA= -github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94= +github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= @@ -914,10 +977,14 @@ github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= +github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -948,6 +1015,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/singnet/snet-ecosystem-contracts v0.1.1 h1:d/xa8T6iFt9efLKATY9SNIu+/lL47Hn9NwszjIEKjnQ= github.com/singnet/snet-ecosystem-contracts v0.1.1/go.mod h1:upRHFLALLPm2chI/tdYGuF/4Kh8RB6rjdVR8HnH27SI= +github.com/singnet/snet-ecosystem-contracts v0.2.1 h1:7zX+b+y1bgCyMexaTSRT1zFvgS0iFYQQ3KlVNUYGs7k= +github.com/singnet/snet-ecosystem-contracts v0.2.1/go.mod h1:upRHFLALLPm2chI/tdYGuF/4Kh8RB6rjdVR8HnH27SI= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -973,16 +1042,17 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -1045,6 +1115,8 @@ github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIf github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.1.2 h1:WQFlrPhpcQl+M2/3dP5cvlTLWPVsL6LGBb9jJt6l/cA= github.com/whyrusleeping/cbor-gen v0.1.2/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= +github.com/whyrusleeping/cbor-gen v0.2.0 h1:v8DREoK/1qQBSc6/UZ4OgU06+9FkywTh8glX0Hi+jkc= +github.com/whyrusleeping/cbor-gen v0.2.0/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= @@ -1072,21 +1144,23 @@ github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= +go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd/api/v3 v3.5.17 h1:cQB8eb8bxwuxOilBpMJAEo8fAONyrdXTHUNcMd8yT1w= -go.etcd.io/etcd/api/v3 v3.5.17/go.mod h1:d1hvkRuXkts6PmaYk2Vrgqbv7H4ADfAKhyJqHNLJCB4= -go.etcd.io/etcd/client/pkg/v3 v3.5.17 h1:XxnDXAWq2pnxqx76ljWwiQ9jylbpC4rvkAeRVOUKKVw= -go.etcd.io/etcd/client/pkg/v3 v3.5.17/go.mod h1:4DqK1TKacp/86nJk4FLQqo6Mn2vvQFBmruW3pP14H/w= -go.etcd.io/etcd/client/v2 v2.305.17 h1:ajFukQfI//xY5VuSeuUw4TJ4WnNR2kAFfV/P0pDdPMs= -go.etcd.io/etcd/client/v2 v2.305.17/go.mod h1:EttKgEgvwikmXN+b7pkEWxDZr6sEaYsqCiS3k4fa/Vg= -go.etcd.io/etcd/client/v3 v3.5.17 h1:o48sINNeWz5+pjy/Z0+HKpj/xSnBkuVhVvXkjEXbqZY= -go.etcd.io/etcd/client/v3 v3.5.17/go.mod h1:j2d4eXTHWkT2ClBgnnEPm/Wuu7jsqku41v9DZ3OtjQo= -go.etcd.io/etcd/pkg/v3 v3.5.17 h1:1k2wZ+oDp41jrk3F9o15o8o7K3/qliBo0mXqxo1PKaE= -go.etcd.io/etcd/pkg/v3 v3.5.17/go.mod h1:FrztuSuaJG0c7RXCOzT08w+PCugh2kCQXmruNYCpCGA= -go.etcd.io/etcd/raft/v3 v3.5.17 h1:wHPW/b1oFBw/+HjDAQ9vfr17OIInejTIsmwMZpK1dNo= -go.etcd.io/etcd/raft/v3 v3.5.17/go.mod h1:uapEfOMPaJ45CqBYIraLO5+fqyIY2d57nFfxzFwy4D4= -go.etcd.io/etcd/server/v3 v3.5.17 h1:xykBwLZk9IdDsB8z8rMdCCPRvhrG+fwvARaGA0TRiyc= -go.etcd.io/etcd/server/v3 v3.5.17/go.mod h1:40sqgtGt6ZJNKm8nk8x6LexZakPu+NDl/DCgZTZ69Cc= +go.etcd.io/etcd/api/v3 v3.5.18 h1:Q4oDAKnmwqTo5lafvB+afbgCDF7E35E4EYV2g+FNGhs= +go.etcd.io/etcd/api/v3 v3.5.18/go.mod h1:uY03Ob2H50077J7Qq0DeehjM/A9S8PhVfbQ1mSaMopU= +go.etcd.io/etcd/client/pkg/v3 v3.5.18 h1:mZPOYw4h8rTk7TeJ5+3udUkfVGBqc+GCjOJYd68QgNM= +go.etcd.io/etcd/client/pkg/v3 v3.5.18/go.mod h1:BxVf2o5wXG9ZJV+/Cu7QNUiJYk4A29sAhoI5tIRsCu4= +go.etcd.io/etcd/client/v2 v2.305.18 h1:jT7ANzlD47yu7t6ZGBr1trUDEN6P0RG9Wnyio6XP2Qo= +go.etcd.io/etcd/client/v2 v2.305.18/go.mod h1:JikXfwJymsNv633PzkAb5xnVZmROgNWr4E68YCEz4jo= +go.etcd.io/etcd/client/v3 v3.5.18 h1:nvvYmNHGumkDjZhTHgVU36A9pykGa2K4lAJ0yY7hcXA= +go.etcd.io/etcd/client/v3 v3.5.18/go.mod h1:kmemwOsPU9broExyhYsBxX4spCTDX3yLgPMWtpBXG6E= +go.etcd.io/etcd/pkg/v3 v3.5.18 h1:ny8rLA18/4AMdrILacOKwt7//TJjc7oS8JIJoLuNvbY= +go.etcd.io/etcd/pkg/v3 v3.5.18/go.mod h1:gb4CDXuN/OgzUgj+VmUFumLYQ2FUMDC6r/plLIjHPI8= +go.etcd.io/etcd/raft/v3 v3.5.18 h1:gueCda+9U76Lvk6rINjNc/mXalUp0u8OK5CVESDZh4I= +go.etcd.io/etcd/raft/v3 v3.5.18/go.mod h1:XBaZHTJt3nLnpS8hMDR55Sxrq76cEC4xWYMBYSY3jcs= +go.etcd.io/etcd/server/v3 v3.5.18 h1:u67DmyYyGOu08OiO9O3wgCSQEjGBNzjhH+FM3BcabcI= +go.etcd.io/etcd/server/v3 v3.5.18/go.mod h1:waeL2uw6TdXniXaus105tiK1aSbblIBi21uk8y7D6Ng= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1096,32 +1170,34 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 h1:FFeLy03iVTXP6ffeN2iXrxfGsZGCjVx0/4KlizjyBwU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0/go.mod h1:TMu73/k1CP8nBUpDLc71Wj/Kf7ZS9FK5b53VapRsP9o= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64= go.opentelemetry.io/otel/exporters/zipkin v1.31.0 h1:CgucL0tj3717DJnni7HVVB2wExzi8c2zJNEA2BhLMvI= go.opentelemetry.io/otel/exporters/zipkin v1.31.0/go.mod h1:rfzOVNiSwIcWtEC2J8epwG26fiaXlYvLySJ7bwsrtAE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1153,6 +1229,8 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= @@ -1175,10 +1253,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1188,8 +1266,10 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= +golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34= +golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1214,8 +1294,10 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1263,10 +1345,6 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1287,8 +1365,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1352,10 +1430,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -1374,16 +1450,14 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= -golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= +golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1422,8 +1496,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1433,6 +1507,8 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= +gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -1474,14 +1550,18 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= -google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb h1:3oy2tynMOP1QbTC0MsNNAV+Se8M2Bd0A5+x1QHyw+pI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= +google.golang.org/genproto v0.0.0-20250127172529-29210b9bc287 h1:WoUI1G0DQ648FKvSl756SKxHQR/bI+y4HyyIQfxMWI8= +google.golang.org/genproto v0.0.0-20250127172529-29210b9bc287/go.mod h1:wkQ2Aj/xvshAUDtO/JHvu9y+AaN9cqs28QuSVSHtZSY= +google.golang.org/genproto v0.0.0-20250207221924-e9438ea467c6 h1:SSk8oMbcHFbMwftDvX4PHbkqss3RkEZUF+k1h9d/sns= +google.golang.org/genproto v0.0.0-20250207221924-e9438ea467c6/go.mod h1:wkQ2Aj/xvshAUDtO/JHvu9y+AaN9cqs28QuSVSHtZSY= +google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287 h1:A2ni10G3UlplFrWdCDJTl7D7mJ7GSRm37S+PDimaKRw= +google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4= +google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 h1:L9JNMl/plZH9wmzQUHleO/ZZDSN+9Gh41wPczNy+5Fk= +google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250204164813-702378808489 h1:5bKytslY8ViY0Cj/ewmRtrWHW64bNF03cAatUUFCdFI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250204164813-702378808489/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 h1:2duwAxN2+k0xLNpjnHTXoMUgnv6VPSp5fiqTuwSxjmI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1501,12 +1581,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.69.0 h1:quSiOM1GJPmPH5XtU+BCoVXcDVJJAzNcoyfC2cCjGkI= -google.golang.org/grpc v1.69.0/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= -google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1519,16 +1595,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= -google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= -google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= -google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index 1efc52f8..667cd3e6 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -268,6 +268,7 @@ func (components *Components) PrepaidUserStorage() storage.TypedAtomicStorage { return components.prepaidUserStorage } + func (components *Components) PaymentChannelService() escrow.PaymentChannelService { if components.paymentChannelService != nil { return components.paymentChannelService @@ -618,7 +619,7 @@ func (components *Components) ModelStorage() *training.ModelStorage { return components.modelStorage } - components.modelStorage = training.NewModelStorage(components.AtomicStorage()) + components.modelStorage = training.NewModelStorage(components.AtomicStorage(), components.OrganizationMetaData()) return components.modelStorage } @@ -638,7 +639,7 @@ func (components *Components) PendingModelStorage() *training.PendingModelStorag return components.pendingModelStorage } - components.pendingModelStorage = training.NewPendingModelStorage(components.AtomicStorage()) + components.pendingModelStorage = training.NewPendingModelStorage(components.AtomicStorage(), components.OrganizationMetaData()) return components.pendingModelStorage } diff --git a/snetd/cmd/serve.go b/snetd/cmd/serve.go index 21d5dd70..2d884218 100644 --- a/snetd/cmd/serve.go +++ b/snetd/cmd/serve.go @@ -285,6 +285,7 @@ func (d *daemon) start() { handler.PrePaidAuthTokenHeader, handler.CurrentBlockNumberHeader, handler.PaymentMultiPartyEscrowAddressHeader, + handler.TrainingModelId, }, }) diff --git a/training/service.go b/training/service.go index eae6d4ce..fa6a3429 100644 --- a/training/service.go +++ b/training/service.go @@ -1,4 +1,4 @@ -//go:generate protoc -I . ./training_daemon.proto ./training_v2.proto --go-grpc_out=. --go_out=. +//go:generate protoc -I . ./training_daemon.proto ./training.proto --go-grpc_out=paths=source_relative:. --go_out=paths=source_relative:. package training @@ -7,6 +7,9 @@ import ( "context" "errors" "fmt" + "github.com/bufbuild/protocompile" + "github.com/bufbuild/protocompile/linker" + "github.com/singnet/snet-daemon/v5/errs" "io" "maps" "net/url" @@ -15,17 +18,10 @@ import ( "sync" "time" - "github.com/bufbuild/protocompile" - "github.com/bufbuild/protocompile/linker" + _ "embed" "github.com/singnet/snet-daemon/v5/blockchain" - "github.com/singnet/snet-daemon/v5/errs" "google.golang.org/grpc/credentials/insecure" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" "google.golang.org/protobuf/types/known/emptypb" - "google.golang.org/protobuf/types/known/structpb" - - _ "embed" "github.com/singnet/snet-daemon/v5/config" "go.uber.org/zap" @@ -37,9 +33,6 @@ const ( DateFormat = "02-01-2006" ) -//go:embed training_v2.proto -var TrainingProtoEmbeded string - // ModelService this is remote AI service provider type ModelService struct { serviceMetaData *blockchain.ServiceMetadata @@ -129,16 +122,8 @@ func (ds *DaemonService) buildPublicModelKey() *PublicModelKey { } } -func (ds *DaemonService) buildPendingModelKey() *PendingModelKey { - return &PendingModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - } -} - func (ds *DaemonService) getPendingModelIds() (*PendingModelData, error) { - key := ds.buildPendingModelKey() + key := ds.pendingStorage.buildPendingModelKey() data, _, err := ds.pendingStorage.Get(key) if err != nil { @@ -152,38 +137,54 @@ func (ds *DaemonService) startUpdateModelStatusWorker(ctx context.Context, model modelKey := ds.storage.buildModelKey(modelId) currentModelData, ok, err := ds.storage.Get(modelKey) if err != nil { - zap.L().Error("err in getting modelData from storage", zap.Error(err)) + zap.L().Error("[startUpdateModelStatusWorker] err in getting modelData from storage", zap.Error(err)) return } if !ok { - zap.L().Error("there is no model with such modelKey", zap.Any("modelKey", modelKey)) + zap.L().Error("[startUpdateModelStatusWorker] there is no model with such modelKey", zap.Any("modelKey", modelKey)) return } _, client, err := ds.getServiceClient() if err != nil { - zap.L().Error("error in gettting service client", zap.Error(err)) + zap.L().Error("[startUpdateModelStatusWorker] error in getting service client", zap.Error(err)) return } response, err := client.GetModelStatus(ctx, &ModelID{ModelId: modelId}) if response == nil || err != nil { - zap.L().Error("error in invoking GetModelStatus, service-provider should implement it", zap.Error(err)) + zap.L().Error("[startUpdateModelStatusWorker] error in invoking GetModelStatus, service-provider should implement it", zap.Error(err)) return } + if response.Status != Status_TRAINING && response.Status != Status_VALIDATING { + err := ds.pendingStorage.RemovePendingModelId(ds.pendingStorage.buildPendingModelKey(), modelId) + if err != nil { + zap.L().Error("[RemovePendingModelId] error in updating model status", zap.Error(err)) + } + } + newModelData := *currentModelData // Shallow copy of the current model data. // However, it does not create deep copies of any slices contained within ModelData; modifications to the slices in newModelData will affect currentModelData. + if currentModelData.Status == response.Status { + // if status don't changed yet we skip it + return + } + newModelData.Status = response.Status - ds.storage.CompareAndSwap(modelKey, currentModelData, &newModelData) + zap.L().Debug("[startUpdateModelStatusWorker]", zap.String("current status", currentModelData.Status.String()), zap.String("new status", response.Status.String())) + ok, err = ds.storage.CompareAndSwap(modelKey, currentModelData, &newModelData) + if !ok || err != nil { + zap.L().Debug("[startUpdateModelStatusWorker] error in updating model status", zap.Bool("isOK", ok), zap.Error(err)) + } } -func (ds *DaemonService) updateModelStatusworker(ctx context.Context, tasks <-chan string, wg *sync.WaitGroup) { +func (ds *DaemonService) updateModelStatusWorker(ctx context.Context, tasks <-chan string, wg *sync.WaitGroup) { defer wg.Done() for { select { case modelID := <-tasks: - ds.startUpdateModelStatusWorker(ctx, modelID) + go ds.startUpdateModelStatusWorker(ctx, modelID) case <-ctx.Done(): return } @@ -192,29 +193,29 @@ func (ds *DaemonService) updateModelStatusworker(ctx context.Context, tasks <-ch func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, interval time.Duration) { ticker := time.NewTicker(interval) - data, err := ds.getPendingModelIds() - if err != nil { - zap.L().Error("Error in getting pending model IDs", zap.Error(err)) - return - } - if data == nil { - zap.L().Debug("There are no pending models") - return - } - tasks := make(chan string, len(data.ModelIDs)) - numWorkers := 3 + + tasks := make(chan string) + //numWorkers := 1docker var wg sync.WaitGroup - for i := 0; i < numWorkers; i++ { - wg.Add(1) - go ds.updateModelStatusworker(ctx, tasks, &wg) - } + //for i := 0; i < numWorkers; i++ { + // wg.Add(1) + go ds.updateModelStatusWorker(ctx, tasks, &wg) + //} defer ticker.Stop() for { select { case <-ticker.C: + data, err := ds.getPendingModelIds() + if err != nil { + zap.L().Error("Error in getting pending model IDs", zap.Error(err)) + return + } + if data == nil { + continue + } for _, modelID := range data.ModelIDs { tasks <- modelID } @@ -247,7 +248,6 @@ func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthVa return price, nil } -// TODO fix func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer) error { var fullData bytes.Buffer var modelID string @@ -265,15 +265,18 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer return err } + var stResp *StatusResponse + zap.L().Debug("UploadAndValidate CALLED") for { req, err := stream.Recv() - if req != nil { - zap.L().Debug("stream.Recv() for model_id " + req.UploadInput.ModelId) + if req == nil { + continue } + zap.L().Debug("stream.Recv() for model_id " + req.UploadInput.ModelId) if err == io.EOF { zap.L().Debug("[UploadAndValidate] EOF") - err := grpcStream.CloseSend() + stResp, err = grpcStream.CloseAndRecv() if err != nil { zap.L().Error("[UploadAndValidate] CloseSend error", zap.Error(err)) } @@ -285,6 +288,12 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer return err } + modelID = req.UploadInput.ModelId + + if modelID == "" { + return errors.New("modelID can't be empty") + } + err = grpcStream.SendMsg(req.UploadInput) if err != nil { zap.L().Error("error in sending upload validation response", zap.Error(err)) @@ -292,14 +301,20 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer } zap.L().Debug("Received chunk of data for model", zap.String("modelID", req.UploadInput.ModelId)) - modelID = req.UploadInput.ModelId fullData.Write(req.UploadInput.Data) } zap.L().Debug("Received file for model %s with size %d bytes", zap.String("modelID", modelID), zap.Int("len", fullData.Len())) closeConn(conn) - return stream.SendAndClose(&StatusResponse{ - Status: Status_VALIDATING, - }) + go func() { + err := ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), modelID) + if err != nil { + zap.L().Error("[AddPendingModelId]", zap.Error(err)) + } + }() + if stResp == nil { + stResp = &StatusResponse{Status: Status_VALIDATING} + } + return stream.SendAndClose(stResp) } func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidateRequest) (*StatusResponse, error) { @@ -309,7 +324,7 @@ func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidat Status: Status_ERRORED, }, WrapError(ErrServiceIssue, err.Error()) } - price, err := client.ValidateModel(ctx, &ValidateRequest{ + statusResp, err := client.ValidateModel(ctx, &ValidateRequest{ ModelId: request.ModelId, TrainingDataLink: request.TrainingDataLink, }) @@ -317,7 +332,14 @@ func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidat if err != nil { return nil, WrapError(ErrServiceIssue, err.Error()) } - return price, nil + go func() { + err := ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), request.ModelId) + if err != nil { + zap.L().Error("[AddPendingModelId]", zap.Error(err)) + } + }() + + return statusResp, nil } func (ds *DaemonService) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { @@ -359,6 +381,12 @@ func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) Status: Status_ERRORED, }, WrapError(ErrServiceIssue, err.Error()) } + go func() { + err := ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), request.ModelId) + if err != nil { + zap.L().Error("[AddPendingModelId]", zap.Error(err)) + } + }() return statusResp, nil } @@ -599,11 +627,7 @@ func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data * oldAddresses = data.AuthorizedAddresses data.AuthorizedAddresses = latestAddresses latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) - // TODO data.IsPublic = request.IsIsPubliclyAccessible data.UpdatedByAddress = request.Authorization.SignerAddress - //if response != nil { - // data.Status = response.Status - //} data.ModelName = request.ModelName data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) data.Description = request.Description @@ -729,15 +753,6 @@ func (ds *DaemonService) updateModelPrices(modelID string, validatePrice, trainP } if validatePrice != nil { data.ValidatePrice = validatePrice.Price - return err -} - -func (ds *DaemonService) getModelKey(modelID string) (key *ModelKey) { - key = &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - ModelId: modelID, } if trainPrice != nil { data.TrainPrice = trainPrice.Price @@ -746,7 +761,7 @@ func (ds *DaemonService) getModelKey(modelID string) (key *ModelKey) { zap.L().Error("[updateModelPrices] issue with updating model data", zap.Error(err)) return fmt.Errorf("[updateModelPrices] issue with updating model data: %s", err) } - return nil + return err } func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (*ModelsResponse, error) { @@ -757,55 +772,81 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque return &ModelsResponse{}, ErrBadAuthorization } - userModelKey := &ModelUserKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - UserAddress: request.Authorization.SignerAddress, + zap.L().Debug("[GetAllModels]", zap.Any("request", request)) + + if request.Statuses == nil || len(request.Statuses) == 0 { + request.Statuses = []Status{Status_CREATED, Status_VALIDATING, Status_VALIDATED, Status_TRAINING, Status_READY_TO_USE, Status_ERRORED, Status_DELETED} } modelDetailsArray := make([]*ModelResponse, 0) - if data, ok, err := ds.userStorage.Get(userModelKey); data != nil && ok && err == nil { - for _, modelId := range data.ModelIds { + + if request.IsPublic == nil || !*request.IsPublic { + + userModelKey := &ModelUserKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + UserAddress: request.Authorization.SignerAddress, + } + + if data, ok, err := ds.userStorage.Get(userModelKey); data != nil && ok && err == nil { modelKey := &ModelKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - ModelId: modelId, } - if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { - boModel := convertModelDataToBO(modelData) - modelDetailsArray = append(modelDetailsArray, boModel) + for _, modelId := range data.ModelIds { + modelKey.ModelId = modelId + if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { + boModel := convertModelDataToBO(modelData) + modelDetailsArray = append(modelDetailsArray, boModel) + } } } } - publicModelKey := &PublicModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - } + if request.IsPublic == nil || *request.IsPublic { - if data, ok, err := ds.publicStorage.Get(publicModelKey); data != nil && ok && err == nil { - for _, modelId := range data.ModelIDs { + publicModelKey := &PublicModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + } + + if data, ok, err := ds.publicStorage.Get(publicModelKey); data != nil && ok && err == nil { modelKey := &ModelKey{ OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), - ModelId: modelId, } - if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { - boModel := convertModelDataToBO(modelData) - modelDetailsArray = append(modelDetailsArray, boModel) + for _, modelId := range data.ModelIDs { + modelKey.ModelId = modelId + if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { + boModel := convertModelDataToBO(modelData) + modelDetailsArray = append(modelDetailsArray, boModel) + } } } } - for _, model := range modelDetailsArray { - zap.L().Debug("Model", zap.String("Name", model.Name)) + filtered := modelDetailsArray[:0] + + for _, v := range modelDetailsArray { + zap.L().Debug("[GetAllModels] model", zap.String("id", v.ModelId), zap.String("status", v.Status.String())) + if strings.Contains(v.Name, request.Name) && + strings.Contains(v.GrpcMethodName, request.GrpcMethodName) && + strings.Contains(v.GrpcServiceName, request.GrpcServiceName) && + strings.Contains(v.CreatedByAddress, request.CreatedByAddress) && + slices.Contains(request.Statuses, v.Status) { + filtered = append(filtered, v) + } + } + + if request.Page != 0 || request.PageSize != 0 { + filtered = filtered[request.Page*request.PageSize : request.PageSize] } - return &ModelsResponse{ListOfModels: modelDetailsArray}, nil + return &ModelsResponse{ListOfModels: filtered}, nil } func (ds *DaemonService) getModelDataToCreate(request *NewModelRequest, response *ModelID) (data *ModelData) { @@ -818,14 +859,13 @@ func (ds *DaemonService) getModelDataToCreate(request *NewModelRequest, response AuthorizedAddresses: request.Model.AddressList, Description: request.Model.Description, ModelName: request.Model.Name, - //TrainingLink: TODO - IsPublic: request.Model.IsPublic, - IsDefault: false, - ModelId: response.ModelId, - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - UpdatedDate: fmt.Sprintf("%v", time.Now().Format(DateFormat)), + IsPublic: request.Model.IsPublic, + IsDefault: false, + ModelId: response.ModelId, + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: ds.organizationMetaData.GetGroupIdString(), + UpdatedDate: fmt.Sprintf("%v", time.Now().Format(DateFormat)), } //by default add the creator to the Authorized list of Address if data.AuthorizedAddresses == nil { @@ -847,6 +887,8 @@ func BuildModelResponse(data *ModelData, status Status) *ModelResponse { TrainingDataLink: data.TrainingLink, Name: data.ModelName, UpdatedDate: data.UpdatedDate, + CreatedDate: data.CreatedDate, + CreatedByAddress: data.CreatedByAddress, } } @@ -888,12 +930,10 @@ func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*St zap.L().Error("issue with deleting ModelId in storage", zap.Error(err)) return response, WrapError(ErrDaemonStorage, fmt.Sprintf("issue with deleting Model %v", err)) } - //responseData := BuildModelResponse(data, response.Status) return &StatusResponse{Status: Status_DELETED}, err } -func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (response *ModelResponse, - err error) { +func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (response *ModelResponse, err error) { if request == nil || request.Authorization == nil { return &ModelResponse{Status: Status_ERRORED}, ErrNoAuthorization } @@ -934,18 +974,33 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re return } +// getFileDescriptors converts text of proto files to bufbuild linker +func getFileDescriptorsWithTraining(protoFiles map[string]string) linker.Files { + protoFiles["training.proto"] = TrainingProtoEmbeded + accessor := protocompile.SourceAccessorFromMap(protoFiles) + r := protocompile.WithStandardImports(&protocompile.SourceResolver{Accessor: accessor}) + compiler := protocompile.Compiler{ + Resolver: r, + SourceInfoMode: protocompile.SourceInfoStandard, + } + fds, err := compiler.Compile(context.Background(), slices.Collect(maps.Keys(protoFiles))...) + if err != nil || fds == nil { + zap.L().Fatal("failed to analyze protofile"+errs.ErrDescURL(errs.InvalidProto), zap.Error(err)) + } + return fds +} + // NewTrainingService daemon self server func NewTrainingService(serMetaData *blockchain.ServiceMetadata, orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage, pendingStorage *PendingModelStorage, publicStorage *PublicModelStorage) DaemonServer { - linkerFiles := getFileDescriptors(serMetaData.ProtoFiles) - serMetaData.ProtoDescriptors = linkerFiles - methodsMD, trainMD, err := parseTrainingMetadata(linkerFiles) + serMetaData.ProtoDescriptors = getFileDescriptorsWithTraining(serMetaData.ProtoFiles) + + methodsMD, trainMD, err := parseTrainingMetadata(serMetaData.ProtoDescriptors) if err != nil { - zap.L().Error(err.Error()) - // TODO - return nil + zap.L().Error("[NewTrainingService] can't init training", zap.Error(err)) + return &NoTrainingDaemonServer{} } serviceURL := config.GetString(config.ModelMaintenanceEndPoint) @@ -963,126 +1018,10 @@ func NewTrainingService(serMetaData *blockchain.ServiceMetadata, methodsMetadata: methodsMD, } - go daemonService.ManageUpdateModelStatusWorkers(context.Background(), 4*time.Second) + go daemonService.ManageUpdateModelStatusWorkers(context.Background(), 3*time.Second) return daemonService } return &NoTrainingDaemonServer{} } - -// parseTrainingMetadata parses metadata from Protobuf files to identify training-related methods -// and their associated metadata. -// Input: -// - protos: a collection of Protobuf files containing definitions of services and methods. -// Output: -// - methodsMD: a map where the key is the combination of service and method names, -// and the value is metadata related to the method (MethodMetadata). -// - trainingMD: a structure containing metadata for training methods, including -// whether training methods are defined and their names grouped by service. -// - err: an error, if any occurred during the parsing process. -func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMetadata, trainingMD *TrainingMetadata, err error) { - methodsMD = make(map[string]*MethodMetadata) - trainingMD = &TrainingMetadata{} - trainingMD.TrainingMethods = make(map[string]*structpb.ListValue) - - for _, protoFile := range protos { - for servicesCounter := 0; servicesCounter < protoFile.Services().Len(); servicesCounter++ { - protoService := protoFile.Services().Get(servicesCounter) - if protoService == nil { - continue - } - for methodsCounter := 0; methodsCounter < protoService.Methods().Len(); methodsCounter++ { - method := protoFile.Services().Get(servicesCounter).Methods().Get(methodsCounter) - if method == nil { - continue - } - inputFields := method.Input().Fields() - if inputFields == nil { - continue - } - for fieldsCounter := 0; fieldsCounter < inputFields.Len(); fieldsCounter++ { - if inputFields.Get(fieldsCounter).Message() != nil { - // if the method accepts modelId, then we consider it as training - if inputFields.Get(fieldsCounter).Message().FullName() == "trainingV2.ModelID" { - // init array if nil - trainingMD.TrainingInProto = true - if trainingMD.TrainingMethods[string(protoService.Name())] == nil { - trainingMD.TrainingMethods[string(protoService.Name())], _ = structpb.NewList(nil) - } - value := structpb.NewStringValue(string(method.Name())) - trainingMD.TrainingMethods[string(protoService.Name())].Values = append(trainingMD.TrainingMethods[string(protoService.Name())].Values, value) - } - } - } - - methodOptions, ok := method.Options().(*descriptorpb.MethodOptions) - if ok && methodOptions != nil { - key := string(protoService.Name() + method.Name()) - methodsMD[key] = &MethodMetadata{} - if proto.HasExtension(methodOptions, E_DatasetDescription) { - if datasetDesc, ok := proto.GetExtension(methodOptions, E_DatasetDescription).(string); ok { - methodsMD[key].DatasetDescription = datasetDesc - } - } - if proto.HasExtension(methodOptions, E_DatasetType) { - if datasetType, ok := proto.GetExtension(methodOptions, E_DatasetType).(string); ok { - methodsMD[key].DatasetType = datasetType - } - } - if proto.HasExtension(methodOptions, E_DatasetFilesType) { - if datasetDesc, ok := proto.GetExtension(methodOptions, E_DatasetFilesType).(string); ok { - methodsMD[key].DatasetFilesType = datasetDesc - } - } - if proto.HasExtension(methodOptions, E_MaxModelsPerUser) { - if datasetDesc, ok := proto.GetExtension(methodOptions, E_MaxModelsPerUser).(uint64); ok { - methodsMD[key].MaxModelsPerUser = datasetDesc - } - } - if proto.HasExtension(methodOptions, E_DefaultModelId) { - if defaultModelId, ok := proto.GetExtension(methodOptions, E_DefaultModelId).(string); ok { - methodsMD[key].DefaultModelId = defaultModelId - } - } - if proto.HasExtension(methodOptions, E_DatasetMaxSizeSingleFileMb) { - if d, ok := proto.GetExtension(methodOptions, E_DatasetMaxSizeSingleFileMb).(uint64); ok { - methodsMD[key].DatasetMaxSizeSingleFileMb = d - } - } - if proto.HasExtension(methodOptions, E_DatasetMaxCountFiles) { - if maxCountFiles, ok := proto.GetExtension(methodOptions, E_DatasetMaxCountFiles).(uint64); ok { - methodsMD[key].DatasetMaxCountFiles = maxCountFiles - } - } - if proto.HasExtension(methodOptions, E_DatasetMaxSizeMb) { - if datasetMaxSizeMb, ok := proto.GetExtension(methodOptions, E_DatasetMaxSizeMb).(uint64); ok { - methodsMD[key].DatasetMaxSizeMb = datasetMaxSizeMb - } - } - if methodsMD[key].DefaultModelId != "" { - zap.L().Debug("training metadata", zap.String("method", string(method.Name())), zap.String("key", key), zap.Any("metadata", methodsMD[key])) - } - } - } - } - } - zap.L().Debug("training methods", zap.Any("methods", trainingMD.TrainingMethods)) - return -} - -// getFileDescriptors converts text of proto file to bufbuild linker -func getFileDescriptors(protoFiles map[string]string) linker.Files { - protoFiles["training_v2.proto"] = TrainingProtoEmbeded - accessor := protocompile.SourceAccessorFromMap(protoFiles) - r := protocompile.WithStandardImports(&protocompile.SourceResolver{Accessor: accessor}) - compiler := protocompile.Compiler{ - Resolver: r, - SourceInfoMode: protocompile.SourceInfoStandard, - } - fds, err := compiler.Compile(context.Background(), slices.Collect(maps.Keys(protoFiles))...) - if err != nil || fds == nil { - zap.L().Fatal("failed to analyze protofile"+errs.ErrDescURL(errs.InvalidProto), zap.Error(err)) - } - return fds -} diff --git a/training/storage.go b/training/storage.go index f97827a7..66964970 100644 --- a/training/storage.go +++ b/training/storage.go @@ -21,7 +21,8 @@ type ModelUserStorage struct { } type PendingModelStorage struct { - delegate storage.TypedAtomicStorage + delegate storage.TypedAtomicStorage + organizationMetaData *blockchain.OrganizationMetaData } type PublicModelStorage struct { @@ -46,13 +47,13 @@ func NewModelStorage(atomicStorage storage.AtomicStorage, orgMetadata *blockchai return &ModelStorage{delegate: modelStorage, organizationMetaData: orgMetadata} } -func NewPendingModelStorage(atomicStorage storage.AtomicStorage) *PendingModelStorage { +func NewPendingModelStorage(atomicStorage storage.AtomicStorage, orgMetadata *blockchain.OrganizationMetaData) *PendingModelStorage { prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/pendingModelStorage") pendingModelStorage := storage.NewTypedAtomicStorageImpl( prefixedStorage, serializePendingModelKey, reflect.TypeOf(PendingModelKey{}), utils.Serialize, utils.Deserialize, reflect.TypeOf(PendingModelData{}), ) - return &PendingModelStorage{delegate: pendingModelStorage} + return &PendingModelStorage{delegate: pendingModelStorage, organizationMetaData: orgMetadata} } func NewPublicModelStorage(atomicStorage storage.AtomicStorage) *PublicModelStorage { @@ -85,7 +86,7 @@ type ModelData struct { Status Status CreatedByAddress string ModelId string - UpdatedByAddress string + UpdatedByAddress string // TODO ?! GroupId string OrganizationId string ServiceId string @@ -94,9 +95,10 @@ type ModelData struct { Description string IsDefault bool TrainingLink string - UpdatedDate string ValidatePrice uint64 TrainPrice uint64 + UpdatedDate string + CreatedDate string } func (data *ModelData) String() string { @@ -289,7 +291,16 @@ func (storage *PendingModelStorage) Put(key *PendingModelKey, state *PendingMode return storage.delegate.Put(key, state) } +func (storage *PendingModelStorage) buildPendingModelKey() *PendingModelKey { + return &PendingModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: storage.organizationMetaData.GetGroupIdString(), + } +} + func (pendingStorage *PendingModelStorage) AddPendingModelId(key *PendingModelKey, modelId string) (err error) { + typedUpdateFunc := func(conditionValues []storage.TypedKeyValueData) (update []storage.TypedKeyValueData, ok bool, err error) { if len(conditionValues) != 1 || conditionValues[0].Key != key { return nil, false, fmt.Errorf("unexpected condition values or missing key") @@ -316,9 +327,69 @@ func (pendingStorage *PendingModelStorage) AddPendingModelId(key *PendingModelKe } } + zap.L().Debug("[AddPendingModelId]", zap.Strings("modelIDS", pendingModelData.ModelIDs)) + // Add the new model ID to the list pendingModelData.ModelIDs = append(pendingModelData.ModelIDs, modelId) + zap.L().Debug("[AddPendingModelId]", zap.Strings("modelIDS", pendingModelData.ModelIDs)) + + // Prepare the updated values for the transaction + newValues := []storage.TypedKeyValueData{ + { + Key: key, + Value: pendingModelData, + Present: true, + }, + } + + return newValues, true, nil + } + + request := storage.TypedCASRequest{ + ConditionKeys: []any{key}, + RetryTillSuccessOrError: true, + Update: typedUpdateFunc, + } + + // Execute the transaction + ok, err := pendingStorage.delegate.ExecuteTransaction(request) + if err != nil { + return fmt.Errorf("transaction execution failed: %w", err) + } + if !ok { + return fmt.Errorf("transaction was not successful") + } + + return nil +} + +func (pendingStorage *PendingModelStorage) RemovePendingModelId(key *PendingModelKey, modelId string) (err error) { + + typedUpdateFunc := func(conditionValues []storage.TypedKeyValueData) (update []storage.TypedKeyValueData, ok bool, err error) { + if len(conditionValues) != 1 || conditionValues[0].Key != key { + return nil, false, fmt.Errorf("unexpected condition values or missing key") + } + + // Fetch the current list of pending model IDs from the storage + currentValue, ok, err := pendingStorage.delegate.Get(key) + if err != nil { + return nil, false, err + } + + var pendingModelData *PendingModelData + if currentValue == nil { + return + } else { + pendingModelData = currentValue.(*PendingModelData) + } + + zap.L().Debug("[RemovePendingModelId]", zap.Strings("modelIDS", pendingModelData.ModelIDs)) + + pendingModelData.ModelIDs = remove(pendingModelData.ModelIDs, modelId) + + zap.L().Debug("[RemovePendingModelId]", zap.Strings("after remove modelIDS", pendingModelData.ModelIDs)) + // Prepare the updated values for the transaction newValues := []storage.TypedKeyValueData{ { diff --git a/training/tests/functional/service_test.go b/training/tests/functional/service_test.go index 85c0b071..524ecf01 100644 --- a/training/tests/functional/service_test.go +++ b/training/tests/functional/service_test.go @@ -80,7 +80,6 @@ func (suite *DaemonServiceSuite) SetupTest() { suite.pendingModelStorage = pendingModelStorage suite.daemonService = training.NewTrainingService( - nil, suite.serviceMetadata, suite.organizationMetadata, modelStorage, @@ -220,9 +219,9 @@ func (suite *DaemonServiceSuite) setupTestConfig() { func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage, *training.PublicModelStorage) { memStorage := storage.NewMemStorage() - modelStorage := training.NewModelStorage(memStorage) + modelStorage := training.NewModelStorage(memStorage, suite.organizationMetadata) userModelStorage := training.NewUserModelStorage(memStorage) - pendingModelStorage := training.NewPendingModelStorage(memStorage) + pendingModelStorage := training.NewPendingModelStorage(memStorage, suite.organizationMetadata) publicModelStorage := training.NewPublicModelStorage(memStorage) modelA := &training.ModelData{ diff --git a/training/tests/integration/daemon_service_test.go b/training/tests/integration/daemon_service_test.go deleted file mode 100644 index e69de29b..00000000 diff --git a/training/tests/unit/storage_test.go b/training/tests/unit/storage_test.go index 5b20a096..0cc77ee8 100644 --- a/training/tests/unit/storage_test.go +++ b/training/tests/unit/storage_test.go @@ -2,6 +2,7 @@ package tests import ( "fmt" + "github.com/singnet/snet-daemon/v5/blockchain" "testing" base_storage "github.com/singnet/snet-daemon/v5/storage" @@ -12,16 +13,17 @@ import ( type ModelStorageSuite struct { suite.Suite - memoryStorage *base_storage.MemoryStorage - storage *training.ModelStorage - userStorage *training.ModelUserStorage - pendingStorage *training.PendingModelStorage - publicStorage *training.PublicModelStorage - organizationId string - serviceId string - groupId string - methodName string - accessibleAddress []string + memoryStorage *base_storage.MemoryStorage + storage *training.ModelStorage + userStorage *training.ModelUserStorage + pendingStorage *training.PendingModelStorage + publicStorage *training.PublicModelStorage + organizationMetaData *blockchain.OrganizationMetaData + organizationId string + serviceId string + groupId string + methodName string + accessibleAddress []string } func (suite *ModelStorageSuite) getModelKey(modelId string) *training.ModelKey { @@ -64,7 +66,6 @@ func (suite *ModelStorageSuite) getUserModelData(modelId []string) *training.Mod OrganizationId: suite.organizationId, ServiceId: suite.serviceId, GroupId: suite.groupId, - // GRPCMethodName: suite.methodName, } } @@ -80,18 +81,25 @@ func (suite *ModelStorageSuite) getPublicModelData(modelIds []string) *training. } } +var testJsonOrgMeta = "{\n \"org_name\": \"semyon_dev\",\n \"org_id\": \"semyon_dev\",\n \"org_type\": \"individual\",\n \"description\": {\n \"description\": \"Describe your organization details here\",\n \"short_description\": \"This is short description of your organization\",\n \"url\": \"https://anyurlofyourorganization\"\n },\n \"assets\": {},\n \"contacts\": [],\n \"groups\": [\n {\n \"group_name\": \"default_group\",\n \"group_id\": \"FtNuizEOUsVCd5f2Fij9soehtRSb58LlTePgkVnsgVI=\",\n \"payment\": {\n \"payment_address\": \"0x747155e03c892B8b311B7Cfbb920664E8c6792fA\",\n \"payment_expiration_threshold\": 40320,\n \"payment_channel_storage_type\": \"etcd\",\n \"payment_channel_storage_client\": {\n \"connection_timeout\": \"10s\",\n \"request_timeout\": \"5s\",\n \"endpoints\": [\n \"http://0.0.0.0:2379\"\n ]\n }\n }\n },\n {\n \"group_name\": \"not_default\",\n \"group_id\": \"udN0SLIvsDdvQQe3Ltv/NwqCh7sPKdz4scYmlI7AMdE=\",\n \"payment\": {\n \"payment_address\": \"0x747155e03c892B8b311B7Cfbb920664E8c6792fA\",\n \"payment_expiration_threshold\": 100,\n \"payment_channel_storage_type\": \"etcd\",\n \"payment_channel_storage_client\": {\n \"connection_timeout\": \"7s\",\n \"request_timeout\": \"5s\",\n \"endpoints\": [\n \"http://0.0.0.0:2379\"\n ]\n }\n }\n }\n ]\n}" + func (suite *ModelStorageSuite) SetupSuite() { + metadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgMeta)) + if err != nil { + panic(err) + } suite.memoryStorage = base_storage.NewMemStorage() - suite.storage = training.NewModelStorage(suite.memoryStorage) + suite.organizationMetaData = metadata + suite.storage = training.NewModelStorage(suite.memoryStorage, suite.organizationMetaData) suite.userStorage = training.NewUserModelStorage(suite.memoryStorage) - suite.pendingStorage = training.NewPendingModelStorage(suite.memoryStorage) + suite.pendingStorage = training.NewPendingModelStorage(suite.memoryStorage, suite.organizationMetaData) suite.publicStorage = training.NewPublicModelStorage(suite.memoryStorage) suite.accessibleAddress = make([]string, 2) suite.accessibleAddress[0] = "ADD1" suite.accessibleAddress[1] = "ADD2" - suite.organizationId = "org_id" - suite.serviceId = "service_id" - suite.groupId = "group_id" + suite.organizationId = "semyon_dev" + suite.serviceId = "semyon_dev" + suite.groupId = "FtNuizEOUsVCd5f2Fij9soehtRSb58LlTePgkVnsgVI=" } func TestFreeCallUserStorageSuite(t *testing.T) { @@ -265,6 +273,34 @@ func (suite *ModelStorageSuite) TestPendingModelStorage_AddPendingModelId() { assert.Equal(suite.T(), data, newData) } +func (suite *ModelStorageSuite) TestPendingModelStorage_AddRemovePendingModelId() { + key := suite.getPendingModelKey() + data := suite.getPendingModelData([]string{"1", "2"}) + + err := suite.pendingStorage.Put(key, data) + assert.NoError(suite.T(), err) + + newData, ok, err := suite.pendingStorage.Get(key) + assert.True(suite.T(), ok) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), data, newData) + + newModelId := "3" + data = suite.getPendingModelData([]string{"1", "2", "3"}) + err = suite.pendingStorage.AddPendingModelId(key, newModelId) + assert.NoError(suite.T(), err) + newData, ok, err = suite.pendingStorage.Get(key) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), data, newData) + + data = suite.getPendingModelData([]string{"2", "3"}) + err = suite.pendingStorage.RemovePendingModelId(key, "1") + assert.NoError(suite.T(), err) + newData, ok, err = suite.pendingStorage.Get(key) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), data, newData) +} + func (suite *ModelStorageSuite) TestPublicModelStorage_Get() { key := suite.getPublicModelKey() data := suite.getPublicModelData([]string{"1", "2"}) diff --git a/training/training_v2.proto b/training/training.proto similarity index 89% rename from training/training_v2.proto rename to training/training.proto index 66007a27..1b6e47b9 100644 --- a/training/training_v2.proto +++ b/training/training.proto @@ -1,7 +1,7 @@ syntax = "proto3"; import "google/protobuf/descriptor.proto"; // Required for indicators to work -package trainingV2; -option go_package = "../training"; +package training; +option go_package = "github.com/singnet/snet-daemon/v5/training"; // Methods that the service provider must implement service Model { @@ -37,19 +37,22 @@ service Model { message ModelResponse { string model_id = 1; Status status = 2; - string updated_date = 3; - string name = 4; - string description = 5; - string grpc_method_name = 6; - string grpc_service_name = 7; + string created_date = 3; + string updated_date = 4; + string name = 5; + string description = 6; + string grpc_method_name = 7; + string grpc_service_name = 8; // List of all addresses that will have access to this model - repeated string address_list = 8; + repeated string address_list = 9; // Access to the model is granted only for use and viewing - bool is_public = 9; + bool is_public = 10; - string training_data_link = 10; + string training_data_link = 11; + + string created_by_address = 12; } // Used as input for new_model requests diff --git a/training/training_daemon.proto b/training/training_daemon.proto index e86ff308..b535f31f 100644 --- a/training/training_daemon.proto +++ b/training/training_daemon.proto @@ -1,10 +1,11 @@ syntax = "proto3"; -package training_daemon; +package training; + import "google/protobuf/descriptor.proto"; // Required for indicators to work import "google/protobuf/struct.proto"; // Required for google.protobuf.ListValue -import "training_v2.proto"; -option go_package = "../training"; +import "training.proto"; import "google/protobuf/empty.proto"; +option go_package = "github.com/singnet/snet-daemon/v5/training"; message AuthorizationDetails { uint64 current_block = 1; // Check for relevance within a range of +/- N blocks @@ -18,7 +19,7 @@ message AuthorizationDetails { message NewModelRequest { AuthorizationDetails authorization = 1; - trainingV2.NewModel model = 2; + training.NewModel model = 2; } message AuthValidateRequest { @@ -29,7 +30,7 @@ message AuthValidateRequest { message UploadAndValidateRequest { AuthorizationDetails authorization = 1; - trainingV2.UploadInput upload_input = 2; + training.UploadInput upload_input = 2; } message CommonRequest { @@ -46,38 +47,38 @@ message UpdateModelRequest { } message ModelsResponse { - repeated trainingV2.ModelResponse list_of_models = 1; + repeated training.ModelResponse list_of_models = 1; } // These methods are implemented only by the daemon; the service provider should ignore them service Daemon { // Free - rpc create_model(NewModelRequest) returns (trainingV2.ModelResponse) {} + rpc create_model(NewModelRequest) returns (training.ModelResponse) {} // Free - rpc validate_model_price(AuthValidateRequest) returns (trainingV2.PriceInBaseUnit) {} + rpc validate_model_price(AuthValidateRequest) returns (training.PriceInBaseUnit) {} // Paid - rpc upload_and_validate(stream UploadAndValidateRequest) returns (trainingV2.StatusResponse) {} + rpc upload_and_validate(stream UploadAndValidateRequest) returns (training.StatusResponse) {} // Paid - rpc validate_model(AuthValidateRequest) returns (trainingV2.StatusResponse) {} + rpc validate_model(AuthValidateRequest) returns (training.StatusResponse) {} // Free, one signature for both train_model_price & train_model methods - rpc train_model_price(CommonRequest) returns (trainingV2.PriceInBaseUnit) {} + rpc train_model_price(CommonRequest) returns (training.PriceInBaseUnit) {} // Paid - rpc train_model(CommonRequest) returns (trainingV2.StatusResponse) {} + rpc train_model(CommonRequest) returns (training.StatusResponse) {} // Free // After deleting the model, the status becomes DELETED in etcd - rpc delete_model(CommonRequest) returns (trainingV2.StatusResponse) {} + rpc delete_model(CommonRequest) returns (training.StatusResponse) {} rpc get_all_models(AllModelsRequest) returns (ModelsResponse) {} - rpc get_model(CommonRequest) returns (trainingV2.ModelResponse) {} + rpc get_model(CommonRequest) returns (training.ModelResponse) {} - rpc update_model(UpdateModelRequest) returns (trainingV2.ModelResponse) {} + rpc update_model(UpdateModelRequest) returns (training.ModelResponse) {} // Unique methods by daemon // One signature for all getters @@ -96,12 +97,15 @@ message MethodMetadataRequest { message AllModelsRequest { AuthorizationDetails authorization = 1; - // Filters - trainingV2.Status status = 2; - bool is_public = 3; - string name = 4; // Search by name - uint64 page_size = 5; // Pagination - uint64 page = 6; + // filters: + repeated training.Status statuses = 3; + optional bool is_public = 4; // null - all, false - only private, true - only public models + string grpc_method_name = 5; + string grpc_service_name = 6; + string name = 7; + string created_by_address = 8; + uint64 page_size = 9; + uint64 page = 10; } message TrainingMetadata { diff --git a/training/util.go b/training/util.go index 5a3310fa..22d730b0 100644 --- a/training/util.go +++ b/training/util.go @@ -2,6 +2,12 @@ package training import ( "bytes" + _ "embed" + "github.com/bufbuild/protocompile/linker" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/known/structpb" "math/big" "github.com/ethereum/go-ethereum/common/math" @@ -67,3 +73,106 @@ func isValuePresent(value string, list []string) bool { } return false } + +//go:embed training.proto +var TrainingProtoEmbeded string + +// parseTrainingMetadata parses metadata from Protobuf files to identify training-related methods +// and their associated metadata. +// Input: +// - protos: a collection of Protobuf files containing definitions of services and methods. +// Output: +// - methodsMD: a map where the key is the combination of service and method names, +// and the value is metadata related to the method (MethodMetadata). +// - trainingMD: a structure containing metadata for training methods, including +// whether training methods are defined and their names grouped by service. +// - err: an error, if any occurred during the parsing process. +func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMetadata, trainingMD *TrainingMetadata, err error) { + methodsMD = make(map[string]*MethodMetadata) + trainingMD = &TrainingMetadata{} + trainingMD.TrainingMethods = make(map[string]*structpb.ListValue) + + for _, protoFile := range protos { + for servicesCounter := 0; servicesCounter < protoFile.Services().Len(); servicesCounter++ { + protoService := protoFile.Services().Get(servicesCounter) + if protoService == nil { + continue + } + for methodsCounter := 0; methodsCounter < protoService.Methods().Len(); methodsCounter++ { + method := protoFile.Services().Get(servicesCounter).Methods().Get(methodsCounter) + if method == nil { + continue + } + inputFields := method.Input().Fields() + if inputFields == nil { + continue + } + for fieldsCounter := 0; fieldsCounter < inputFields.Len(); fieldsCounter++ { + if inputFields.Get(fieldsCounter).Message() != nil { + // if the method accepts modelId, then we consider it as training + if inputFields.Get(fieldsCounter).Message().FullName() == "training.ModelID" { + // init array if nil + trainingMD.TrainingInProto = true + if trainingMD.TrainingMethods[string(protoService.Name())] == nil { + trainingMD.TrainingMethods[string(protoService.Name())], _ = structpb.NewList(nil) + } + value := structpb.NewStringValue(string(method.Name())) + trainingMD.TrainingMethods[string(protoService.Name())].Values = append(trainingMD.TrainingMethods[string(protoService.Name())].Values, value) + } + } + } + + methodOptions, ok := method.Options().(*descriptorpb.MethodOptions) + if ok && methodOptions != nil { + key := string(protoService.Name() + method.Name()) + methodsMD[key] = &MethodMetadata{} + if proto.HasExtension(methodOptions, E_DatasetDescription) { + if datasetDesc, ok := proto.GetExtension(methodOptions, E_DatasetDescription).(string); ok { + methodsMD[key].DatasetDescription = datasetDesc + } + } + if proto.HasExtension(methodOptions, E_DatasetType) { + if datasetType, ok := proto.GetExtension(methodOptions, E_DatasetType).(string); ok { + methodsMD[key].DatasetType = datasetType + } + } + if proto.HasExtension(methodOptions, E_DatasetFilesType) { + if datasetDesc, ok := proto.GetExtension(methodOptions, E_DatasetFilesType).(string); ok { + methodsMD[key].DatasetFilesType = datasetDesc + } + } + if proto.HasExtension(methodOptions, E_MaxModelsPerUser) { + if datasetDesc, ok := proto.GetExtension(methodOptions, E_MaxModelsPerUser).(uint64); ok { + methodsMD[key].MaxModelsPerUser = datasetDesc + } + } + if proto.HasExtension(methodOptions, E_DefaultModelId) { + if defaultModelId, ok := proto.GetExtension(methodOptions, E_DefaultModelId).(string); ok { + methodsMD[key].DefaultModelId = defaultModelId + } + } + if proto.HasExtension(methodOptions, E_DatasetMaxSizeSingleFileMb) { + if d, ok := proto.GetExtension(methodOptions, E_DatasetMaxSizeSingleFileMb).(uint64); ok { + methodsMD[key].DatasetMaxSizeSingleFileMb = d + } + } + if proto.HasExtension(methodOptions, E_DatasetMaxCountFiles) { + if maxCountFiles, ok := proto.GetExtension(methodOptions, E_DatasetMaxCountFiles).(uint64); ok { + methodsMD[key].DatasetMaxCountFiles = maxCountFiles + } + } + if proto.HasExtension(methodOptions, E_DatasetMaxSizeMb) { + if datasetMaxSizeMb, ok := proto.GetExtension(methodOptions, E_DatasetMaxSizeMb).(uint64); ok { + methodsMD[key].DatasetMaxSizeMb = datasetMaxSizeMb + } + } + if methodsMD[key].DefaultModelId != "" { + zap.L().Debug("training metadata", zap.String("method", string(method.Name())), zap.String("key", key), zap.Any("metadata", methodsMD[key])) + } + } + } + } + } + zap.L().Debug("training methods", zap.Any("methods", trainingMD.TrainingMethods)) + return +} From ff6ba8781effb501cbe0fdf34d68ae2fcf210862 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Mon, 10 Feb 2025 18:54:45 +0300 Subject: [PATCH 14/22] fix tests --- escrow/income.go | 2 +- escrow/income_test.go | 16 ++++++++-------- ipfsutils/ipfsutils_test.go | 4 ++-- pricing/pricing_strategy_test.go | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/escrow/income.go b/escrow/income.go index 6e645698..94bfa007 100644 --- a/escrow/income.go +++ b/escrow/income.go @@ -49,7 +49,7 @@ func (validator *incomeStreamValidator) Validate(data *IncomeStreamData) (err er price := big.NewInt(0) - if data.GrpcContext.Info.FullMethod == "/training_daemon.Daemon/upload_and_validate" { + if data.GrpcContext != nil && data.GrpcContext.Info.FullMethod == "/training_daemon.Daemon/upload_and_validate" { modelID, ok := data.GrpcContext.MD[handler.TrainingModelId] if !ok { return errors.New("no training model found") diff --git a/escrow/income_test.go b/escrow/income_test.go index b2f7424f..0584958c 100644 --- a/escrow/income_test.go +++ b/escrow/income_test.go @@ -5,6 +5,7 @@ import ( "github.com/singnet/snet-daemon/v5/blockchain" "github.com/singnet/snet-daemon/v5/handler" "github.com/singnet/snet-daemon/v5/pricing" + "google.golang.org/grpc" "math/big" "testing" @@ -15,7 +16,7 @@ type incomeValidatorMockType struct { err error } -func (incomeValidator *incomeValidatorMockType) Validate(income *IncomeData) (err error) { +func (incomeValidator *incomeValidatorMockType) Validate(income *IncomeStreamData) (err error) { return incomeValidator.err } @@ -39,20 +40,20 @@ func TestIncomeValidate(t *testing.T) { pricingStrt, err := pricing.InitPricingStrategy(pricingMetadata) assert.Nil(t, err) pricingStrt.AddPricingTypes(&MockPriceType{}) - incomeValidator := NewIncomeValidator(pricingStrt) + incomeValidator := NewIncomeStreamValidator(pricingStrt, nil) price := big.NewInt(0) income.Sub(price, one) - err = incomeValidator.Validate(&IncomeData{Income: income}) + err = incomeValidator.Validate(&IncomeStreamData{Income: income, GrpcContext: &handler.GrpcStreamContext{Info: &grpc.StreamServerInfo{FullMethod: "test"}}}) msg := fmt.Sprintf("income %s does not equal to price %s", income, price) assert.Equal(t, NewPaymentError(Unauthenticated, msg), err) income.Set(price) - err = incomeValidator.Validate(&IncomeData{Income: income}) + err = incomeValidator.Validate(&IncomeStreamData{Income: income, GrpcContext: &handler.GrpcStreamContext{Info: &grpc.StreamServerInfo{FullMethod: "test"}}}) assert.Nil(t, err) income.Add(price, one) - err = incomeValidator.Validate(&IncomeData{Income: income}) + err = incomeValidator.Validate(&IncomeStreamData{Income: income, GrpcContext: &handler.GrpcStreamContext{Info: &grpc.StreamServerInfo{FullMethod: "test"}}}) msg = fmt.Sprintf("income %s does not equal to price %s", income, price) assert.Equal(t, NewPaymentError(Unauthenticated, msg), err) } @@ -71,8 +72,7 @@ func TestIncomeValidateForPriceError(t *testing.T) { pricingStrt, err := pricing.InitPricingStrategy(pricingMetadata) assert.Nil(t, err) pricingStrt.AddPricingTypes(&MockPriceErrorType{}) - incomeValidator := NewIncomeValidator(pricingStrt) - err = incomeValidator.Validate(&IncomeData{Income: big.NewInt(0)}) + incomeValidator := NewIncomeStreamValidator(pricingStrt, nil) + err = incomeValidator.Validate(&IncomeStreamData{Income: big.NewInt(0), GrpcContext: &handler.GrpcStreamContext{Info: &grpc.StreamServerInfo{FullMethod: "test"}}}) assert.Equal(t, err.Error(), "Error in Determining Price") - } diff --git a/ipfsutils/ipfsutils_test.go b/ipfsutils/ipfsutils_test.go index ca7ee09b..847b5844 100644 --- a/ipfsutils/ipfsutils_test.go +++ b/ipfsutils/ipfsutils_test.go @@ -46,7 +46,7 @@ func (suite *IpfsUtilsTestSuite) TestReadFiles() { assert.Nil(suite.T(), err) assert.NotNil(suite.T(), protoFiles) - excpectedProtoFiles := []string{`syntax = "proto3"; + expectedProtoFiles := map[string]string{"example_service.proto": `syntax = "proto3"; package example_service; @@ -66,5 +66,5 @@ service Calculator { rpc div(Numbers) returns (Result) {} }`} - assert.Equal(suite.T(), excpectedProtoFiles, protoFiles) + assert.EqualValues(suite.T(), expectedProtoFiles, protoFiles) } diff --git a/pricing/pricing_strategy_test.go b/pricing/pricing_strategy_test.go index 21050fdd..12ccca7f 100644 --- a/pricing/pricing_strategy_test.go +++ b/pricing/pricing_strategy_test.go @@ -2,7 +2,9 @@ package pricing import ( "github.com/singnet/snet-daemon/v5/blockchain" + "github.com/singnet/snet-daemon/v5/handler" "github.com/stretchr/testify/assert" + "google.golang.org/grpc" "math/big" "testing" ) @@ -14,8 +16,7 @@ func TestPricing_GetPrice(t *testing.T) { pricing, err := InitPricingStrategy(metadata) if pricing != nil { - - price, err := pricing.GetPrice(nil) + price, err := pricing.GetPrice(&handler.GrpcStreamContext{Info: &grpc.StreamServerInfo{FullMethod: "add"}}) assert.Equal(t, price, big.NewInt(2)) assert.Nil(t, err) } @@ -24,5 +25,4 @@ func TestPricing_GetPrice(t *testing.T) { pricing,err = InitPricingStrategy(metadata) assert.Equal(t,err.Error(),"No PricingStrategy strategy defined in Metadata ") assert.Nil(t,pricing)*/ - } From 7714305c5b48def30d0a9b3f4603660b9bf3f660 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Mon, 10 Feb 2025 19:31:22 +0300 Subject: [PATCH 15/22] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb3c710b..ca577652 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: - name: install protoc (protobuf) uses: arduino/setup-protoc@v3 with: - version: "27.2" + version: "29.3" include-pre-releases: false - name: chmod +x From dafa256f34624148eb82a56ad1ba26d312ca746a Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Thu, 20 Feb 2025 09:56:48 +0300 Subject: [PATCH 16/22] auth for training & fix bugs --- escrow/income.go | 12 +- handler/unary_interceptor.go | 5 +- snetd/cmd/components.go | 6 +- snetd/cmd/serve.go | 4 +- training/service.go | 398 +++++++++++++--------- training/storage.go | 54 +-- training/{tests/unit => }/storage_test.go | 64 ++-- training/tests/functional/service_test.go | 9 +- training/training.proto | 1 + training/training_daemon.proto | 13 +- training/util.go | 86 ++++- training/util_test.go | 146 ++++++++ 12 files changed, 557 insertions(+), 241 deletions(-) rename training/{tests/unit => }/storage_test.go (85%) create mode 100644 training/util_test.go diff --git a/escrow/income.go b/escrow/income.go index 94bfa007..95729837 100644 --- a/escrow/income.go +++ b/escrow/income.go @@ -7,6 +7,7 @@ import ( "github.com/singnet/snet-daemon/v5/training" "go.uber.org/zap" "math/big" + "strings" "github.com/singnet/snet-daemon/v5/handler" ) @@ -49,7 +50,7 @@ func (validator *incomeStreamValidator) Validate(data *IncomeStreamData) (err er price := big.NewInt(0) - if data.GrpcContext != nil && data.GrpcContext.Info.FullMethod == "/training_daemon.Daemon/upload_and_validate" { + if data.GrpcContext != nil && strings.Contains(data.GrpcContext.Info.FullMethod, "/upload_and_validate") { modelID, ok := data.GrpcContext.MD[handler.TrainingModelId] if !ok { return errors.New("no training model found") @@ -114,10 +115,13 @@ func (validator *trainUnaryValidator) Validate(data *IncomeUnaryData) (err error price := big.NewInt(0) - switch data.GrpcContext.Info.FullMethod { - case "/training_daemon.Daemon/train_model": + lastSlash := strings.LastIndex(data.GrpcContext.Info.FullMethod, "/") + methodName := data.GrpcContext.Info.FullMethod[lastSlash+1:] + + switch methodName { + case "train_model": price = price.SetUint64(model.TrainPrice) - case "/training_daemon.Daemon/validate_model": + case "validate_model": price = price.SetUint64(model.ValidatePrice) default: return nil diff --git a/handler/unary_interceptor.go b/handler/unary_interceptor.go index d778629d..4db744ad 100644 --- a/handler/unary_interceptor.go +++ b/handler/unary_interceptor.go @@ -8,6 +8,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" + "strings" ) type GrpcUnaryContext struct { @@ -39,8 +40,10 @@ type paymentValidationUnaryInterceptor struct { func (interceptor *paymentValidationUnaryInterceptor) unaryIntercept(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, e error) { var err *GrpcError + ctx = context.WithValue(ctx, "method", info.FullMethod) + // pass non training requests and free requests - if info.FullMethod != "/training_daemon.Daemon/validate_model" && info.FullMethod != "/training_daemon.Daemon/train_model" { + if !strings.Contains(info.FullMethod, "validate_model") && !strings.Contains(info.FullMethod, "train_model") { resp, e := handler(ctx, req) if e != nil { zap.L().Warn("gRPC handler returned error", zap.Error(e)) diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index 667cd3e6..d11e673e 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -629,7 +629,7 @@ func (components *Components) ModelUserStorage() *training.ModelUserStorage { return components.modelUserStorage } - components.modelUserStorage = training.NewUserModelStorage(components.AtomicStorage()) + components.modelUserStorage = training.NewUserModelStorage(components.AtomicStorage(), components.organizationMetaData) return components.modelUserStorage } @@ -649,7 +649,7 @@ func (components *Components) PublicModelStorage() *training.PublicModelStorage return components.publicModelStorage } - components.publicModelStorage = training.NewPublicModelStorage(components.AtomicStorage()) + components.publicModelStorage = training.NewPublicModelStorage(components.AtomicStorage(), components.OrganizationMetaData()) return components.publicModelStorage } @@ -663,7 +663,7 @@ func (components *Components) TrainingService() training.DaemonServer { return components.trainingService } - components.trainingService = training.NewTrainingService(components.ServiceMetaData(), + components.trainingService = training.NewTrainingService(components.blockchain, components.ServiceMetaData(), components.OrganizationMetaData(), components.ModelStorage(), components.ModelUserStorage(), components.PendingModelStorage(), components.PublicModelStorage()) return components.trainingService } diff --git a/snetd/cmd/serve.go b/snetd/cmd/serve.go index 2d884218..a0e526d9 100644 --- a/snetd/cmd/serve.go +++ b/snetd/cmd/serve.go @@ -268,8 +268,8 @@ func (d *daemon) start() { AllowOriginFunc: func(origin string) bool { return true }, - ExposedHeaders: []string{"X-Grpc-Web", "Content-Length", "Access-Control-Allow-Origin", "Content-Type", "Origin"}, - AllowedHeaders: []string{"X-Grpc-Web", "User-Agent", "Origin", "Accept", "Authorization", "Content-Type", "X-Requested-With", "Content-Length", "Access-Control-Allow-Origin", + ExposedHeaders: []string{"X-Grpc-Web", "Content-Length", "Access-Control-Allow-Origin", "Content-Type", "Origin", "Grpc-Status", "Grpc-Message"}, + AllowedHeaders: []string{"Grpc-Status", "Grpc-Message", "X-Grpc-Web", "User-Agent", "Origin", "Accept", "Authorization", "Content-Type", "X-Requested-With", "Content-Length", "Access-Control-Allow-Origin", handler.PaymentTypeHeader, handler.ClientTypeHeader, handler.PaymentChannelSignatureHeader, diff --git a/training/service.go b/training/service.go index fa6a3429..edab529f 100644 --- a/training/service.go +++ b/training/service.go @@ -33,18 +33,10 @@ const ( DateFormat = "02-01-2006" ) -// ModelService this is remote AI service provider -type ModelService struct { - serviceMetaData *blockchain.ServiceMetadata - organizationMetaData *blockchain.OrganizationMetaData - storage *ModelStorage - userStorage *ModelUserStorage - serviceUrl string -} - type DaemonService struct { serviceMetaData *blockchain.ServiceMetadata organizationMetaData *blockchain.OrganizationMetaData + blockchain *blockchain.Processor storage *ModelStorage userStorage *ModelUserStorage pendingStorage *PendingModelStorage @@ -54,15 +46,13 @@ type DaemonService struct { methodsMetadata map[string]*MethodMetadata } -func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest) (*ModelResponse, error) { - - zap.L().Debug("CreateModel request") +func (ds *DaemonService) CreateModel(ctx context.Context, request *NewModelRequest) (*ModelResponse, error) { if request == nil || request.Authorization == nil { return &ModelResponse{Status: Status_ERRORED}, ErrNoAuthorization } - if err := ds.verifySignature(request.Authorization); err != nil { + if err := ds.verifySignature(request.Authorization, ctx.Value("method")); err != nil { zap.L().Error("unable to create model, bad authorization provided", zap.Error(err)) return &ModelResponse{Status: Status_ERRORED}, ErrBadAuthorization } @@ -85,7 +75,7 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest return &ModelResponse{Status: Status_ERRORED}, WrapError(ErrServiceInvocation, err.Error()) } - responseModelID, errClient := client.CreateModel(c, request.Model) + responseModelID, errClient := client.CreateModel(ctx, request.Model) closeConn(conn) if errClient != nil { zap.L().Error("[CreateModel] unable to call CreateModel", zap.Error(errClient)) @@ -114,14 +104,6 @@ func (ds *DaemonService) CreateModel(c context.Context, request *NewModelRequest return modelResponse, err } -func (ds *DaemonService) buildPublicModelKey() *PublicModelKey { - return &PublicModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - } -} - func (ds *DaemonService) getPendingModelIds() (*PendingModelData, error) { key := ds.pendingStorage.buildPendingModelKey() @@ -164,17 +146,15 @@ func (ds *DaemonService) startUpdateModelStatusWorker(ctx context.Context, model } } - newModelData := *currentModelData // Shallow copy of the current model data. - // However, it does not create deep copies of any slices contained within ModelData; modifications to the slices in newModelData will affect currentModelData. if currentModelData.Status == response.Status { // if status don't changed yet we skip it return } - newModelData.Status = response.Status + currentModelData.Status = response.Status zap.L().Debug("[startUpdateModelStatusWorker]", zap.String("current status", currentModelData.Status.String()), zap.String("new status", response.Status.String())) - ok, err = ds.storage.CompareAndSwap(modelKey, currentModelData, &newModelData) - if !ok || err != nil { + err = ds.storage.Put(modelKey, currentModelData) + if err != nil { zap.L().Debug("[startUpdateModelStatusWorker] error in updating model status", zap.Bool("isOK", ok), zap.Error(err)) } } @@ -226,21 +206,39 @@ func (ds *DaemonService) ManageUpdateModelStatusWorkers(ctx context.Context, int } } -func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthValidateRequest) (*PriceInBaseUnit, error) { +func (ds *DaemonService) ValidateModelPrice(ctx context.Context, req *AuthValidateRequest) (*PriceInBaseUnit, error) { + + if req == nil || req.Authorization == nil { + return nil, ErrNoAuthorization + } + + if req.ModelId == "" { + return nil, ErrEmptyModelID + } + + if err := ds.verifySignature(req.Authorization, ctx.Value("method")); err != nil { + return nil, WrapError(ErrAccessToModel, err.Error()) + } + + if err := ds.verifyCreatedByAddress(req.ModelId, req.Authorization.SignerAddress); err != nil { + return nil, WrapError(ErrAccessToModel, err.Error()) + } + conn, client, err := ds.getServiceClient() if client == nil || err != nil { return nil, WrapError(ErrServiceIssue, err.Error()) } + price, err := client.ValidateModelPrice(ctx, &ValidateRequest{ - ModelId: request.ModelId, - TrainingDataLink: request.TrainingDataLink, + ModelId: req.ModelId, + TrainingDataLink: req.TrainingDataLink, }) closeConn(conn) if err != nil || price == nil { zap.L().Error("[ValidateModelPrice] service issue", zap.Error(err)) return nil, WrapError(ErrServiceIssue, err.Error()) } - err = ds.updateModelPrices(request.ModelId, price, nil) + err = ds.updateModelPrices(req.ModelId, price, nil) if err != nil { zap.L().Debug("[ValidateModelPrice] can't update model prices") return nil, err @@ -248,18 +246,16 @@ func (ds *DaemonService) ValidateModelPrice(ctx context.Context, request *AuthVa return price, nil } -func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer) error { +func (ds *DaemonService) UploadAndValidate(clientStream Daemon_UploadAndValidateServer) error { var fullData bytes.Buffer var modelID string - zap.L().Debug("start upload and validate") - - conn, client, err := ds.getServiceClient() + providerConn, client, err := ds.getServiceClient() if err != nil { zap.L().Debug(err.Error()) return err } - grpcStream, err := client.UploadAndValidate(context.Background()) + providerStream, err := client.UploadAndValidate(context.Background()) if err != nil { zap.L().Error("error in sending UploadAndValidate", zap.Error(err)) return err @@ -267,44 +263,58 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer var stResp *StatusResponse - zap.L().Debug("UploadAndValidate CALLED") for { - req, err := stream.Recv() - if req == nil { - continue - } - zap.L().Debug("stream.Recv() for model_id " + req.UploadInput.ModelId) + req, err := clientStream.Recv() if err == io.EOF { - zap.L().Debug("[UploadAndValidate] EOF") - stResp, err = grpcStream.CloseAndRecv() + zap.L().Debug("[UploadAndValidate] received EOF") + stResp, err = providerStream.CloseAndRecv() if err != nil { - zap.L().Error("[UploadAndValidate] CloseSend error", zap.Error(err)) + zap.L().Error("[UploadAndValidate] providerStream.CloseAndRecv() error", zap.Error(err)) } break } + if req == nil { + continue + } + zap.L().Debug("[UploadAndValidate] received", zap.String("modelID", req.UploadInput.ModelId)) if err != nil { - zap.L().Debug("req is nil?", zap.Bool("bool", req == nil)) - zap.L().Error("error in receiving upload request", zap.Error(err)) + zap.L().Debug("[UploadAndValidate]", zap.Bool("req is nil", req == nil)) + zap.L().Error("[UploadAndValidate] error in receiving upload request", zap.Error(err)) return err } + if req.Authorization == nil { + providerStream.CloseSend() + return ErrNoAuthorization + } + + if err := ds.verifySignature(req.Authorization, "upload_and_validate"); err != nil { + providerStream.CloseSend() + return WrapError(ErrNoAuthorization, err.Error()) + } + + if err := ds.verifyCreatedByAddress(req.UploadInput.ModelId, req.Authorization.SignerAddress); err != nil { + providerStream.CloseSend() + return WrapError(ErrAccessToModel, err.Error()) + } + modelID = req.UploadInput.ModelId if modelID == "" { return errors.New("modelID can't be empty") } - err = grpcStream.SendMsg(req.UploadInput) + err = providerStream.SendMsg(req.UploadInput) if err != nil { - zap.L().Error("error in sending upload validation response", zap.Error(err)) + zap.L().Error("[UploadAndValidate] error in sending upload validation response", zap.Error(err)) return err } - zap.L().Debug("Received chunk of data for model", zap.String("modelID", req.UploadInput.ModelId)) + zap.L().Debug("[UploadAndValidate] Received chunk of data for model", zap.String("modelID", req.UploadInput.ModelId)) fullData.Write(req.UploadInput.Data) } - zap.L().Debug("Received file for model %s with size %d bytes", zap.String("modelID", modelID), zap.Int("len", fullData.Len())) - closeConn(conn) + zap.L().Debug("[UploadAndValidate] Received file for model %s with size %d bytes", zap.String("modelID", modelID), zap.Int("len", fullData.Len())) + closeConn(providerConn) go func() { err := ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), modelID) if err != nil { @@ -314,26 +324,65 @@ func (ds *DaemonService) UploadAndValidate(stream Daemon_UploadAndValidateServer if stResp == nil { stResp = &StatusResponse{Status: Status_VALIDATING} } - return stream.SendAndClose(stResp) + _, err = ds.updateModelStatus(modelID, stResp.Status) + if err != nil { + zap.L().Error("[UploadAndValidate] updateModelStatus", zap.Error(err)) + } + err = clientStream.SendAndClose(stResp) + if err != nil { + zap.L().Error("[UploadAndValidate] clientStream.SendAndClose() error", zap.Error(err)) + } + return err } -func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidateRequest) (*StatusResponse, error) { +func (ds *DaemonService) ValidateModel(ctx context.Context, req *AuthValidateRequest) (*StatusResponse, error) { + + if req == nil || req.Authorization == nil { + return nil, ErrNoAuthorization + } + + if req.ModelId == "" { + return nil, ErrEmptyModelID + } + + if err := ds.verifySignature(req.Authorization, ctx.Value("method")); err != nil { + return nil, WrapError(ErrAccessToModel, err.Error()) + } + + if err := ds.verifyCreatedByAddress(req.ModelId, req.Authorization.SignerAddress); err != nil { + return &StatusResponse{}, + WrapError(ErrAccessToModel, err.Error()) + } + conn, client, err := ds.getServiceClient() if client == nil || err != nil { return &StatusResponse{ Status: Status_ERRORED, }, WrapError(ErrServiceIssue, err.Error()) } + + model, err := ds.storage.GetModel(req.ModelId) + if err != nil { + return nil, WrapError(ErrModelDoesntExist, err.Error()) + } statusResp, err := client.ValidateModel(ctx, &ValidateRequest{ - ModelId: request.ModelId, - TrainingDataLink: request.TrainingDataLink, + ModelId: req.ModelId, + TrainingDataLink: req.TrainingDataLink, }) closeConn(conn) if err != nil { return nil, WrapError(ErrServiceIssue, err.Error()) } + key := ds.storage.buildModelKey(req.ModelId) + model.TrainingLink = req.TrainingDataLink + model.Status = statusResp.Status + err = ds.storage.Put(key, model) + if err != nil { + zap.L().Error("Error in putting data in storage", zap.Error(err)) + } + go func() { - err := ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), request.ModelId) + err := ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), req.ModelId) if err != nil { zap.L().Error("[AddPendingModelId]", zap.Error(err)) } @@ -342,20 +391,36 @@ func (ds *DaemonService) ValidateModel(ctx context.Context, request *AuthValidat return statusResp, nil } -func (ds *DaemonService) TrainModelPrice(ctx context.Context, request *CommonRequest) (*PriceInBaseUnit, error) { +func (ds *DaemonService) TrainModelPrice(ctx context.Context, req *CommonRequest) (*PriceInBaseUnit, error) { + + if req == nil || req.Authorization == nil { + return nil, ErrNoAuthorization + } + + if req.ModelId == "" { + return nil, ErrEmptyModelID + } + + if err := ds.verifySignature(req.Authorization, ctx.Value("method")); err != nil { + return nil, WrapError(ErrAccessToModel, err.Error()) + } + + if err := ds.verifyCreatedByAddress(req.ModelId, req.Authorization.SignerAddress); err != nil { + return nil, WrapError(ErrAccessToModel, err.Error()) + } conn, client, err := ds.getServiceClient() if client == nil || err != nil { return nil, WrapError(ErrServiceIssue, err.Error()) } price, err := client.TrainModelPrice(ctx, &ModelID{ - ModelId: request.ModelId, + ModelId: req.ModelId, }) closeConn(conn) if err != nil { zap.L().Debug("[TrainModelPrice] can't update model prices") return nil, WrapError(ErrServiceIssue, err.Error()) } - err = ds.updateModelPrices(request.ModelId, nil, price) + err = ds.updateModelPrices(req.ModelId, nil, price) if err != nil { zap.L().Debug("[TrainModelPrice] can't update model prices") return nil, err @@ -363,7 +428,23 @@ func (ds *DaemonService) TrainModelPrice(ctx context.Context, request *CommonReq return price, nil } -func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) (*StatusResponse, error) { +func (ds *DaemonService) TrainModel(ctx context.Context, req *CommonRequest) (*StatusResponse, error) { + if req == nil || req.Authorization == nil { + return &StatusResponse{Status: Status_ERRORED}, ErrNoAuthorization + } + + if req.ModelId == "" { + return &StatusResponse{Status: Status_ERRORED}, ErrEmptyModelID + } + + if err := ds.verifySignature(req.Authorization, ctx.Value("method")); err != nil { + return &StatusResponse{Status: Status_ERRORED}, + WrapError(ErrAccessToModel, err.Error()) + } + if err := ds.verifyCreatedByAddress(req.ModelId, req.Authorization.SignerAddress); err != nil { + return nil, WrapError(ErrAccessToModel, err.Error()) + } + conn, client, err := ds.getServiceClient() if client == nil || err != nil { zap.L().Error("issue with service", zap.Error(err)) @@ -372,7 +453,7 @@ func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) }, WrapError(ErrServiceIssue, err.Error()) } statusResp, err := client.TrainModel(ctx, &ModelID{ - ModelId: request.ModelId, + ModelId: req.ModelId, }) closeConn(conn) if err != nil { @@ -382,7 +463,12 @@ func (ds *DaemonService) TrainModel(ctx context.Context, request *CommonRequest) }, WrapError(ErrServiceIssue, err.Error()) } go func() { - err := ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), request.ModelId) + _, err = ds.updateModelStatus(req.ModelId, statusResp.Status) + if err != nil { + zap.L().Error("Error in updating model data in storage", zap.Error(err)) + } + + err = ds.pendingStorage.AddPendingModelId(ds.pendingStorage.buildPendingModelKey(), req.ModelId) if err != nil { zap.L().Error("[AddPendingModelId]", zap.Error(err)) } @@ -394,28 +480,28 @@ func (ds *DaemonService) GetTrainingMetadata(ctx context.Context, empty *emptypb return ds.trainingMetadata, nil } -func (ds *DaemonService) UpdateModel(ctx context.Context, request *UpdateModelRequest) (*ModelResponse, error) { +func (ds *DaemonService) UpdateModel(ctx context.Context, req *UpdateModelRequest) (*ModelResponse, error) { - if request == nil || request.Authorization == nil { + if req == nil || req.Authorization == nil { return &ModelResponse{Status: Status_ERRORED}, ErrNoAuthorization } - if err := ds.verifySignature(request.Authorization); err != nil { + if err := ds.verifySignature(req.Authorization, ctx.Value("method")); err != nil { return &ModelResponse{Status: Status_ERRORED}, WrapError(ErrAccessToModel, err.Error()) } - if err := ds.verifySignerHasAccessToTheModel(request.ModelId, request.Authorization.SignerAddress); err != nil { + if err := ds.verifySignerHasAccessToTheModel(req.ModelId, req.Authorization.SignerAddress); err != nil { return &ModelResponse{}, WrapError(ErrAccessToModel, err.Error()) } - if request.ModelId == "" { + if req.ModelId == "" { return &ModelResponse{Status: Status_ERRORED}, ErrEmptyModelID } - if err := ds.verifyCreatedByAddress(request.ModelId, request.Authorization.SignerAddress); err != nil { + if err := ds.verifyCreatedByAddress(req.ModelId, req.Authorization.SignerAddress); err != nil { return &ModelResponse{}, err } zap.L().Info("Updating model") - data, err := ds.updateModelDetails(request) + data, err := ds.updateModelDetails(req) if err != nil || data == nil { return &ModelResponse{Status: Status_ERRORED}, WrapError(ErrDaemonStorage, err.Error()) @@ -518,7 +604,7 @@ func (ds *DaemonService) createModelDetails(request *NewModelRequest, response * } if request.Model.IsPublic { - publicModelKey := ds.buildPublicModelKey() + publicModelKey := ds.publicStorage.buildPublicModelKey() err := ds.publicStorage.AddPublicModelId(publicModelKey, response.ModelId) if err != nil { zap.L().Error("error in adding model id to public model storage") @@ -534,7 +620,7 @@ func getModelUserKey(key *ModelKey, address string) *ModelUserKey { OrganizationId: key.OrganizationId, ServiceId: key.ServiceId, GroupId: key.GroupId, - UserAddress: address, + UserAddress: strings.ToLower(address), } } @@ -597,69 +683,80 @@ func (ds *DaemonService) deleteModelDetails(req *CommonRequest) (data *ModelData func convertModelDataToBO(data *ModelData) (responseData *ModelResponse) { responseData = &ModelResponse{ + Status: data.Status, ModelId: data.ModelId, + Name: data.ModelName, GrpcMethodName: data.GRPCMethodName, GrpcServiceName: data.GRPCServiceName, Description: data.Description, IsPublic: data.IsPublic, AddressList: data.AuthorizedAddresses, TrainingDataLink: data.TrainingLink, - Name: data.ModelName, - //OrganizationId: data.OrganizationId, - //ServiceId: data.ServiceId, - //GroupId: data.GroupId, - UpdatedDate: data.UpdatedDate, - Status: data.Status, + UpdatedDate: data.UpdatedDate, + CreatedDate: data.CreatedDate, + UpdatedByAddress: data.UpdatedByAddress, + CreatedByAddress: data.CreatedByAddress, } return } func (ds *DaemonService) updateModelDetails(request *UpdateModelRequest) (data *ModelData, err error) { + if data, err = ds.storage.GetModel(request.ModelId); err != nil || data == nil { + return nil, errors.New("can't find model to update") + } + + oldAddresses := make([]string, len(data.AuthorizedAddresses)) + copy(oldAddresses, data.AuthorizedAddresses) + + if request.AddressList != nil && len(request.AddressList) > 0 { + + // By default, add the creator to the Authorized list of addresses + if !sliceContainsEqualFold(data.AuthorizedAddresses, data.CreatedByAddress) { + request.AddressList = append(request.AddressList, strings.ToLower(request.Authorization.SignerAddress)) + } + + data.AuthorizedAddresses = request.AddressList + } + + if request.ModelName != nil { + data.ModelName = *request.ModelName + } + if request.Description != nil { + data.Description = *request.Description + } + + data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) + data.UpdatedByAddress = request.Authorization.SignerAddress + key := ds.storage.buildModelKey(request.ModelId) - oldAddresses := make([]string, 0) - var latestAddresses []string - // by default add the creator to the Authorized list of Address - if request.AddressList != nil || len(request.AddressList) > 0 { - latestAddresses = request.AddressList - } - latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) // add creator - if data, err = ds.storage.GetModel(request.ModelId); err == nil && data != nil { - oldAddresses = data.AuthorizedAddresses - data.AuthorizedAddresses = latestAddresses - latestAddresses = append(latestAddresses, request.Authorization.SignerAddress) - data.UpdatedByAddress = request.Authorization.SignerAddress - data.ModelName = request.ModelName - data.UpdatedDate = fmt.Sprintf("%v", time.Now().Format(DateFormat)) - data.Description = request.Description - - err = ds.storage.Put(key, data) - if err != nil { - zap.L().Error("Error in putting data in user storage", zap.Error(err)) + err = ds.storage.Put(key, data) + if err != nil { + zap.L().Error("Error in putting data in user storage", zap.Error(err)) + } + + //get the difference of all the addresses blockchain/w old and new + updatedAddresses := difference(oldAddresses, request.AddressList) + for _, address := range updatedAddresses { + modelUserKey := getModelUserKey(key, address) + modelUserData := ds.getModelUserData(key, address) + //if the address is present in the request but not in the old address , add it to the storage + if sliceContainsEqualFold(request.AddressList, address) { + modelUserData.ModelIds = append(modelUserData.ModelIds, request.ModelId) + } else { // the address was present in the old data , but not in new , hence needs to be deleted + modelUserData.ModelIds = remove(modelUserData.ModelIds, request.ModelId) } - //get the difference of all the addresses b/w old and new - updatedAddresses := difference(oldAddresses, latestAddresses) - for _, address := range updatedAddresses { - modelUserKey := getModelUserKey(key, address) - modelUserData := ds.getModelUserData(key, address) - //if the address is present in the request but not in the old address , add it to the storage - if isValuePresent(address, request.AddressList) { - modelUserData.ModelIds = append(modelUserData.ModelIds, request.ModelId) - } else { // the address was present in the old data , but not in new , hence needs to be deleted - modelUserData.ModelIds = remove(modelUserData.ModelIds, request.ModelId) - } - err = ds.userStorage.Put(modelUserKey, modelUserData) - if err != nil { - zap.L().Error("Error in putting data in storage", zap.Error(err)) - } + err = ds.userStorage.Put(modelUserKey, modelUserData) + if err != nil { + zap.L().Error("Error in putting data in storage", zap.Error(err)) } } - return + return data, err } // ensure only authorized use func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address string) (err error) { // check if model is public - publicModelKey := ds.buildPublicModelKey() + publicModelKey := ds.publicStorage.buildPublicModelKey() publicModels, ok, err := ds.publicStorage.Get(publicModelKey) if ok && err == nil { if slices.Contains(publicModels.ModelIDs, modelId) { @@ -668,14 +765,8 @@ func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address } // check user access if model is not public - userModelKey := &ModelUserKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - UserAddress: address, - } - userModelsData, ok, err := ds.userStorage.Get(userModelKey) - + modelUserKey := ds.userStorage.buildModelUserKey(address) + userModelsData, ok, err := ds.userStorage.Get(modelUserKey) if err != nil { return WrapError(ErrGetUserModelStorage, err.Error()) } @@ -693,23 +784,17 @@ func (ds *DaemonService) verifySignerHasAccessToTheModel(modelId string, address // ensure only owner can update the model state func (ds *DaemonService) verifyCreatedByAddress(modelId, address string) (err error) { - modelKey := &ModelKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - ModelId: modelId, - } - modelData, ok, err := ds.storage.Get(modelKey) + modelData, ok, err := ds.storage.Get(ds.storage.buildModelKey(modelId)) if err != nil { return WrapError(ErrGetModelStorage, err.Error()) } if !ok { - return WrapError(ErrGetModelStorage, fmt.Sprintf("model data doesn't for key: %s", modelKey)) + return WrapError(ErrGetModelStorage, fmt.Sprintf("model data doesn't exist for modelID: %s", modelId)) } - if modelData.CreatedByAddress != address { + if !strings.EqualFold(modelData.CreatedByAddress, address) { return ErrNotOwnerModel } @@ -764,11 +849,12 @@ func (ds *DaemonService) updateModelPrices(modelID string, validatePrice, trainP return err } -func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsRequest) (*ModelsResponse, error) { +func (ds *DaemonService) GetAllModels(ctx context.Context, request *AllModelsRequest) (*ModelsResponse, error) { if request == nil || request.Authorization == nil { return &ModelsResponse{}, ErrNoAuthorization } - if err := ds.verifySignature(request.Authorization); err != nil { + + if err := ds.verifySignature(request.Authorization, ctx.Value("method")); err != nil { return &ModelsResponse{}, ErrBadAuthorization } @@ -798,8 +884,10 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque for _, modelId := range data.ModelIds { modelKey.ModelId = modelId if modelData, modelOk, modelErr := ds.storage.Get(modelKey); modelOk && modelData != nil && modelErr == nil { - boModel := convertModelDataToBO(modelData) - modelDetailsArray = append(modelDetailsArray, boModel) + if !modelData.IsPublic { + boModel := convertModelDataToBO(modelData) + modelDetailsArray = append(modelDetailsArray, boModel) + } } } } @@ -832,18 +920,18 @@ func (ds *DaemonService) GetAllModels(c context.Context, request *AllModelsReque filtered := modelDetailsArray[:0] for _, v := range modelDetailsArray { - zap.L().Debug("[GetAllModels] model", zap.String("id", v.ModelId), zap.String("status", v.Status.String())) + //zap.L().Debug("[GetAllModels] model", zap.String("id", v.ModelId), zap.String("status", v.Status.String())) if strings.Contains(v.Name, request.Name) && strings.Contains(v.GrpcMethodName, request.GrpcMethodName) && strings.Contains(v.GrpcServiceName, request.GrpcServiceName) && - strings.Contains(v.CreatedByAddress, request.CreatedByAddress) && + strings.Contains(v.CreatedByAddress, strings.ToLower(request.CreatedByAddress)) && slices.Contains(request.Statuses, v.Status) { filtered = append(filtered, v) } } if request.Page != 0 || request.PageSize != 0 { - filtered = filtered[request.Page*request.PageSize : request.PageSize] + filtered = paginate(filtered, int(request.Page), int(request.PageSize)) } return &ModelsResponse{ListOfModels: filtered}, nil @@ -854,24 +942,23 @@ func (ds *DaemonService) getModelDataToCreate(request *NewModelRequest, response Status: Status_CREATED, GRPCServiceName: request.Model.GrpcServiceName, GRPCMethodName: request.Model.GrpcMethodName, - CreatedByAddress: request.Authorization.SignerAddress, - UpdatedByAddress: request.Authorization.SignerAddress, + CreatedByAddress: strings.ToLower(request.Authorization.SignerAddress), + UpdatedByAddress: strings.ToLower(request.Authorization.SignerAddress), AuthorizedAddresses: request.Model.AddressList, Description: request.Model.Description, ModelName: request.Model.Name, IsPublic: request.Model.IsPublic, - IsDefault: false, ModelId: response.ModelId, OrganizationId: config.GetString(config.OrganizationId), ServiceId: config.GetString(config.ServiceId), GroupId: ds.organizationMetaData.GetGroupIdString(), UpdatedDate: fmt.Sprintf("%v", time.Now().Format(DateFormat)), + CreatedDate: fmt.Sprintf("%v", time.Now().Format(DateFormat)), } - //by default add the creator to the Authorized list of Address - if data.AuthorizedAddresses == nil { - data.AuthorizedAddresses = make([]string, 0) + // By default, add the creator to the Authorized list of addresses + if !sliceContainsEqualFold(data.AuthorizedAddresses, data.CreatedByAddress) { + data.AuthorizedAddresses = append(data.AuthorizedAddresses, strings.ToLower(data.CreatedByAddress)) } - data.AuthorizedAddresses = append(data.AuthorizedAddresses, data.CreatedByAddress) return } @@ -889,10 +976,11 @@ func BuildModelResponse(data *ModelData, status Status) *ModelResponse { UpdatedDate: data.UpdatedDate, CreatedDate: data.CreatedDate, CreatedByAddress: data.CreatedByAddress, + UpdatedByAddress: data.UpdatedByAddress, } } -func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*StatusResponse, error) { +func (ds *DaemonService) DeleteModel(ctx context.Context, req *CommonRequest) (*StatusResponse, error) { if req == nil || req.Authorization == nil { return &StatusResponse{Status: Status_ERRORED}, ErrNoAuthorization @@ -902,12 +990,12 @@ func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*St return &StatusResponse{Status: Status_ERRORED}, ErrEmptyModelID } - if err := ds.verifySignature(req.Authorization); err != nil { + if err := ds.verifySignature(req.Authorization, ctx.Value("method")); err != nil { return &StatusResponse{Status: Status_ERRORED}, WrapError(ErrAccessToModel, err.Error()) } - if err := ds.verifySignerHasAccessToTheModel(req.ModelId, req.Authorization.SignerAddress); err != nil { + if err := ds.verifyCreatedByAddress(req.ModelId, req.Authorization.SignerAddress); err != nil { return &StatusResponse{}, WrapError(ErrAccessToModel, err.Error()) } @@ -933,11 +1021,11 @@ func (ds *DaemonService) DeleteModel(c context.Context, req *CommonRequest) (*St return &StatusResponse{Status: Status_DELETED}, err } -func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (response *ModelResponse, err error) { +func (ds *DaemonService) GetModel(ctx context.Context, request *CommonRequest) (response *ModelResponse, err error) { if request == nil || request.Authorization == nil { return &ModelResponse{Status: Status_ERRORED}, ErrNoAuthorization } - if err = ds.verifySignature(request.Authorization); err != nil { + if err = ds.verifySignature(request.Authorization, ctx.Value("method")); err != nil { return &ModelResponse{Status: Status_ERRORED}, ErrBadAuthorization } if request.ModelId == "" { @@ -957,8 +1045,8 @@ func (ds *DaemonService) GetModel(c context.Context, request *CommonRequest) (re zap.L().Error("error in invoking GetModelStatus, service-provider should realize it", zap.Error(err)) return &ModelResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking GetModelStatus, service-provider should realize it") } - zap.L().Info("[GetModelStatus] response from service-provider", zap.Any("response", responseStatus)) - zap.L().Debug("[GetModelStatus] updating model status based on response from UpdateModel") + zap.L().Info("[GetModelStatus] response from service-provider", zap.Any("status", responseStatus.Status)) + zap.L().Debug("[GetModelStatus] updating model status based on response from GetModelStatus") data, err := ds.updateModelStatus(request.ModelId, responseStatus.Status) closeConn(conn) zap.L().Debug("[GetModelStatus] data that be returned to client", zap.Any("data", data)) @@ -991,7 +1079,7 @@ func getFileDescriptorsWithTraining(protoFiles map[string]string) linker.Files { } // NewTrainingService daemon self server -func NewTrainingService(serMetaData *blockchain.ServiceMetadata, +func NewTrainingService(b *blockchain.Processor, serMetaData *blockchain.ServiceMetadata, orgMetadata *blockchain.OrganizationMetaData, storage *ModelStorage, userStorage *ModelUserStorage, pendingStorage *PendingModelStorage, publicStorage *PublicModelStorage) DaemonServer { @@ -1002,11 +1090,15 @@ func NewTrainingService(serMetaData *blockchain.ServiceMetadata, zap.L().Error("[NewTrainingService] can't init training", zap.Error(err)) return &NoTrainingDaemonServer{} } + if trainMD.TrainingInProto && config.GetBool(config.ModelTrainingEnabled) { + trainMD.TrainingEnabled = true + } serviceURL := config.GetString(config.ModelMaintenanceEndPoint) if config.IsValidUrl(serviceURL) && config.GetBool(config.BlockchainEnabledKey) { daemonService := &DaemonService{ + blockchain: b, serviceMetaData: serMetaData, organizationMetaData: orgMetadata, storage: storage, diff --git a/training/storage.go b/training/storage.go index 66964970..29ccd094 100644 --- a/training/storage.go +++ b/training/storage.go @@ -6,6 +6,7 @@ import ( "github.com/singnet/snet-daemon/v5/config" "go.uber.org/zap" "reflect" + "strings" "github.com/singnet/snet-daemon/v5/storage" "github.com/singnet/snet-daemon/v5/utils" @@ -17,7 +18,8 @@ type ModelStorage struct { } type ModelUserStorage struct { - delegate storage.TypedAtomicStorage + delegate storage.TypedAtomicStorage + organizationMetaData *blockchain.OrganizationMetaData } type PendingModelStorage struct { @@ -26,16 +28,17 @@ type PendingModelStorage struct { } type PublicModelStorage struct { - delegate storage.TypedAtomicStorage + delegate storage.TypedAtomicStorage + organizationMetaData *blockchain.OrganizationMetaData } -func NewUserModelStorage(atomicStorage storage.AtomicStorage) *ModelUserStorage { +func NewUserModelStorage(atomicStorage storage.AtomicStorage, orgMetadata *blockchain.OrganizationMetaData) *ModelUserStorage { prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/userModelStorage") userModelStorage := storage.NewTypedAtomicStorageImpl( prefixedStorage, serializeModelUserKey, reflect.TypeOf(ModelUserKey{}), utils.Serialize, utils.Deserialize, reflect.TypeOf(ModelUserData{}), ) - return &ModelUserStorage{delegate: userModelStorage} + return &ModelUserStorage{delegate: userModelStorage, organizationMetaData: orgMetadata} } func NewModelStorage(atomicStorage storage.AtomicStorage, orgMetadata *blockchain.OrganizationMetaData) *ModelStorage { @@ -56,22 +59,20 @@ func NewPendingModelStorage(atomicStorage storage.AtomicStorage, orgMetadata *bl return &PendingModelStorage{delegate: pendingModelStorage, organizationMetaData: orgMetadata} } -func NewPublicModelStorage(atomicStorage storage.AtomicStorage) *PublicModelStorage { +func NewPublicModelStorage(atomicStorage storage.AtomicStorage, orgMetadata *blockchain.OrganizationMetaData) *PublicModelStorage { prefixedStorage := storage.NewPrefixedAtomicStorage(atomicStorage, "/model-user/publicModelStorage") publicModelStorage := storage.NewTypedAtomicStorageImpl( prefixedStorage, serializePublicModelKey, reflect.TypeOf(PublicModelKey{}), utils.Serialize, utils.Deserialize, reflect.TypeOf(PublicModelData{}), ) - return &PublicModelStorage{delegate: publicModelStorage} + return &PublicModelStorage{delegate: publicModelStorage, organizationMetaData: orgMetadata} } type ModelKey struct { OrganizationId string ServiceId string GroupId string - //GRPCMethodName string - //GRPCServiceName string - ModelId string + ModelId string } func (key *ModelKey) String() string { @@ -80,20 +81,19 @@ func (key *ModelKey) String() string { } type ModelData struct { + ModelId string IsPublic bool + Status Status ModelName string AuthorizedAddresses []string - Status Status CreatedByAddress string - ModelId string - UpdatedByAddress string // TODO ?! + UpdatedByAddress string GroupId string OrganizationId string ServiceId string GRPCMethodName string GRPCServiceName string Description string - IsDefault bool TrainingLink string ValidatePrice uint64 TrainPrice uint64 @@ -102,10 +102,9 @@ type ModelData struct { } func (data *ModelData) String() string { - return fmt.Sprintf("{DATA:%v|%v|%v|%v|%v|%v|IsPublic:%v|accesibleAddress:%v|createdBy:%v|updatedBy:%v|status:%v|TrainingLin:%v}", - data.OrganizationId, - data.ServiceId, data.GroupId, data.GRPCServiceName, data.GRPCMethodName, data.ModelId, data.AuthorizedAddresses, data.IsPublic, - data.CreatedByAddress, data.UpdatedByAddress, data.Status, data.TrainingLink) + return fmt.Sprintf("{DATA:%v|%v|%v|%v|%v|%v|Name:%v|IsPublic:%v|AuthorizedAddresses:%v|CreatedBy:%v|UpdatedBy:%v|Status:%v|TrainingLink:%v|Updated:%v|Created:%v|ValPrice:%v|TrPrice:%v|Desc:%v}", + data.OrganizationId, data.ServiceId, data.GroupId, data.GRPCServiceName, data.GRPCMethodName, data.ModelId, data.ModelName, data.IsPublic, data.AuthorizedAddresses, + data.CreatedByAddress, data.UpdatedByAddress, data.Status, data.TrainingLink, data.UpdatedDate, data.CreatedDate, data.ValidatePrice, data.TrainPrice, data.Description) } type ModelUserKey struct { @@ -127,9 +126,7 @@ type ModelUserData struct { OrganizationId string ServiceId string GroupId string - //GRPCMethodName string - //GRPCServiceName string - UserAddress string + UserAddress string } func (data *ModelUserData) String() string { @@ -234,6 +231,15 @@ func serializeModelUserKey(key any) (serialized string, err error) { return modelUserKey.String(), nil } +func (storage *ModelUserStorage) buildModelUserKey(address string) *ModelUserKey { + return &ModelUserKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: storage.organizationMetaData.GetGroupIdString(), + UserAddress: strings.ToLower(address), + } +} + func (storage *ModelUserStorage) Get(key *ModelUserKey) (state *ModelUserData, ok bool, err error) { value, ok, err := storage.delegate.Get(key) if err != nil || !ok { @@ -524,3 +530,11 @@ func (storage *PublicModelStorage) CompareAndSwap(key *PublicModelKey, prevState newState *PublicModelData) (ok bool, err error) { return storage.delegate.CompareAndSwap(key, prevState, newState) } + +func (storage *PublicModelStorage) buildPublicModelKey() *PublicModelKey { + return &PublicModelKey{ + OrganizationId: config.GetString(config.OrganizationId), + ServiceId: config.GetString(config.ServiceId), + GroupId: storage.organizationMetaData.GetGroupIdString(), + } +} diff --git a/training/tests/unit/storage_test.go b/training/storage_test.go similarity index 85% rename from training/tests/unit/storage_test.go rename to training/storage_test.go index 0cc77ee8..550916b8 100644 --- a/training/tests/unit/storage_test.go +++ b/training/storage_test.go @@ -1,23 +1,21 @@ -package tests +package training import ( "fmt" "github.com/singnet/snet-daemon/v5/blockchain" - "testing" - base_storage "github.com/singnet/snet-daemon/v5/storage" - "github.com/singnet/snet-daemon/v5/training" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "testing" ) type ModelStorageSuite struct { suite.Suite memoryStorage *base_storage.MemoryStorage - storage *training.ModelStorage - userStorage *training.ModelUserStorage - pendingStorage *training.PendingModelStorage - publicStorage *training.PublicModelStorage + storage *ModelStorage + userStorage *ModelUserStorage + pendingStorage *PendingModelStorage + publicStorage *PublicModelStorage organizationMetaData *blockchain.OrganizationMetaData organizationId string serviceId string @@ -26,29 +24,29 @@ type ModelStorageSuite struct { accessibleAddress []string } -func (suite *ModelStorageSuite) getModelKey(modelId string) *training.ModelKey { - return &training.ModelKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, +func (suite *ModelStorageSuite) getModelKey(modelId string) *ModelKey { + return &ModelKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, ServiceId: suite.serviceId, ModelId: modelId} } -func (suite *ModelStorageSuite) getUserModelKey(address string) *training.ModelUserKey { - return &training.ModelUserKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, +func (suite *ModelStorageSuite) getUserModelKey(address string) *ModelUserKey { + return &ModelUserKey{OrganizationId: suite.organizationId, GroupId: suite.groupId, ServiceId: suite.serviceId, UserAddress: address} } -func (suite *ModelStorageSuite) getPendingModelKey() *training.PendingModelKey { - return &training.PendingModelKey{OrganizationId: suite.organizationId, ServiceId: suite.serviceId, +func (suite *ModelStorageSuite) getPendingModelKey() *PendingModelKey { + return &PendingModelKey{OrganizationId: suite.organizationId, ServiceId: suite.serviceId, GroupId: suite.groupId} } -func (suite *ModelStorageSuite) getPublicModelKey() *training.PublicModelKey { - return &training.PublicModelKey{OrganizationId: suite.organizationId, ServiceId: suite.serviceId, +func (suite *ModelStorageSuite) getPublicModelKey() *PublicModelKey { + return &PublicModelKey{OrganizationId: suite.organizationId, ServiceId: suite.serviceId, GroupId: suite.groupId} } -func (suite *ModelStorageSuite) getModelData(modelId string) *training.ModelData { - return &training.ModelData{ - Status: training.Status_CREATED, +func (suite *ModelStorageSuite) getModelData(modelId string) *ModelData { + return &ModelData{ + Status: Status_CREATED, ModelId: modelId, OrganizationId: suite.organizationId, ServiceId: suite.serviceId, @@ -60,8 +58,8 @@ func (suite *ModelStorageSuite) getModelData(modelId string) *training.ModelData } } -func (suite *ModelStorageSuite) getUserModelData(modelId []string) *training.ModelUserData { - return &training.ModelUserData{ +func (suite *ModelStorageSuite) getUserModelData(modelId []string) *ModelUserData { + return &ModelUserData{ ModelIds: modelId, OrganizationId: suite.organizationId, ServiceId: suite.serviceId, @@ -69,14 +67,14 @@ func (suite *ModelStorageSuite) getUserModelData(modelId []string) *training.Mod } } -func (suite *ModelStorageSuite) getPendingModelData(modelIds []string) *training.PendingModelData { - return &training.PendingModelData{ +func (suite *ModelStorageSuite) getPendingModelData(modelIds []string) *PendingModelData { + return &PendingModelData{ ModelIDs: modelIds, } } -func (suite *ModelStorageSuite) getPublicModelData(modelIds []string) *training.PublicModelData { - return &training.PublicModelData{ +func (suite *ModelStorageSuite) getPublicModelData(modelIds []string) *PublicModelData { + return &PublicModelData{ ModelIDs: modelIds, } } @@ -90,10 +88,10 @@ func (suite *ModelStorageSuite) SetupSuite() { } suite.memoryStorage = base_storage.NewMemStorage() suite.organizationMetaData = metadata - suite.storage = training.NewModelStorage(suite.memoryStorage, suite.organizationMetaData) - suite.userStorage = training.NewUserModelStorage(suite.memoryStorage) - suite.pendingStorage = training.NewPendingModelStorage(suite.memoryStorage, suite.organizationMetaData) - suite.publicStorage = training.NewPublicModelStorage(suite.memoryStorage) + suite.storage = NewModelStorage(suite.memoryStorage, suite.organizationMetaData) + suite.userStorage = NewUserModelStorage(suite.memoryStorage, suite.organizationMetaData) + suite.pendingStorage = NewPendingModelStorage(suite.memoryStorage, suite.organizationMetaData) + suite.publicStorage = NewPublicModelStorage(suite.memoryStorage, suite.organizationMetaData) suite.accessibleAddress = make([]string, 2) suite.accessibleAddress[0] = "ADD1" suite.accessibleAddress[1] = "ADD2" @@ -141,7 +139,7 @@ func (suite *ModelStorageSuite) TestModelStorage_PutIfAbsent() { assert.Equal(suite.T(), err, nil) } -func (suite *ModelStorageSuite) Test_serializeModelKey() { +func (suite *ModelStorageSuite) TestSerializeModelKey() { modelId := "1" expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId, modelId) @@ -151,7 +149,7 @@ func (suite *ModelStorageSuite) Test_serializeModelKey() { assert.Equal(suite.T(), expectedSerializedKey, serializedKey) } -func (suite *ModelStorageSuite) Test_serializeUserModelKey() { +func (suite *ModelStorageSuite) TestSerializeUserModelKey() { userAddress := "test_address" expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId, userAddress) @@ -161,7 +159,7 @@ func (suite *ModelStorageSuite) Test_serializeUserModelKey() { assert.Equal(suite.T(), expectedSerializedKey, serializedKey) } -func (suite *ModelStorageSuite) Test_serializePendingModelKey() { +func (suite *ModelStorageSuite) TestSerializePendingModelKey() { expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId) key := suite.getPendingModelKey() @@ -170,7 +168,7 @@ func (suite *ModelStorageSuite) Test_serializePendingModelKey() { assert.Equal(suite.T(), expectedSerializedKey, serializedKey) } -func (suite *ModelStorageSuite) Test_serializePublicModelKey() { +func (suite *ModelStorageSuite) TestSerializePublicModelKey() { expectedSerializedKey := fmt.Sprintf("{ID:%v|%v|%v}", suite.organizationId, suite.serviceId, suite.groupId) key := suite.getPublicModelKey() diff --git a/training/tests/functional/service_test.go b/training/tests/functional/service_test.go index 524ecf01..16807fea 100644 --- a/training/tests/functional/service_test.go +++ b/training/tests/functional/service_test.go @@ -26,6 +26,7 @@ import ( type DaemonServiceSuite struct { suite.Suite + blockchain *blockchain.Processor modelStorage *training.ModelStorage userModelStorage *training.ModelUserStorage pendingModelStorage *training.PendingModelStorage @@ -80,6 +81,7 @@ func (suite *DaemonServiceSuite) SetupTest() { suite.pendingModelStorage = pendingModelStorage suite.daemonService = training.NewTrainingService( + suite.blockchain, suite.serviceMetadata, suite.organizationMetadata, modelStorage, @@ -220,9 +222,9 @@ func (suite *DaemonServiceSuite) setupTestConfig() { func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage, *training.PublicModelStorage) { memStorage := storage.NewMemStorage() modelStorage := training.NewModelStorage(memStorage, suite.organizationMetadata) - userModelStorage := training.NewUserModelStorage(memStorage) + userModelStorage := training.NewUserModelStorage(memStorage, suite.organizationMetadata) pendingModelStorage := training.NewPendingModelStorage(memStorage, suite.organizationMetadata) - publicModelStorage := training.NewPublicModelStorage(memStorage) + publicModelStorage := training.NewPublicModelStorage(memStorage, suite.organizationMetadata) modelA := &training.ModelData{ IsPublic: true, @@ -238,7 +240,6 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr GRPCMethodName: "string", GRPCServiceName: "string", Description: "string", - IsDefault: true, TrainingLink: "string", UpdatedDate: "string", } @@ -264,7 +265,6 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr GRPCMethodName: "string", GRPCServiceName: "string", Description: "string", - IsDefault: true, TrainingLink: "string", UpdatedDate: "string", } @@ -290,7 +290,6 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr GRPCMethodName: "string", GRPCServiceName: "string", Description: "string", - IsDefault: true, TrainingLink: "string", UpdatedDate: "string", } diff --git a/training/training.proto b/training/training.proto index 1b6e47b9..8582c666 100644 --- a/training/training.proto +++ b/training/training.proto @@ -53,6 +53,7 @@ message ModelResponse { string training_data_link = 11; string created_by_address = 12; + string updated_by_address = 13; } // Used as input for new_model requests diff --git a/training/training_daemon.proto b/training/training_daemon.proto index b535f31f..b7315c56 100644 --- a/training/training_daemon.proto +++ b/training/training_daemon.proto @@ -8,11 +8,12 @@ import "google/protobuf/empty.proto"; option go_package = "github.com/singnet/snet-daemon/v5/training"; message AuthorizationDetails { - uint64 current_block = 1; // Check for relevance within a range of +/- N blocks - // Signer can specify any message here + // Check for relevance within a range of +/- N blocks + uint64 current_block = 1; + // Signer should specify method name in message string message = 2; // Signature of the following message: - // ("user specified message", user_address, current_block_number) + // ("message", user_address, current_block_number) bytes signature = 3; string signer_address = 4; } @@ -41,8 +42,8 @@ message CommonRequest { message UpdateModelRequest { AuthorizationDetails authorization = 1; string model_id = 2; - string model_name = 3; - string description = 4; + optional string model_name = 3; + optional string description = 4; repeated string address_list = 5; } @@ -98,7 +99,7 @@ message MethodMetadataRequest { message AllModelsRequest { AuthorizationDetails authorization = 1; // filters: - repeated training.Status statuses = 3; + repeated training.Status statuses = 3; // optional optional bool is_public = 4; // null - all, false - only private, true - only public models string grpc_method_name = 5; string grpc_service_name = 6; diff --git a/training/util.go b/training/util.go index 22d730b0..ddac3057 100644 --- a/training/util.go +++ b/training/util.go @@ -3,22 +3,62 @@ package training import ( "bytes" _ "embed" + "errors" + "fmt" "github.com/bufbuild/protocompile/linker" + "github.com/singnet/snet-daemon/v5/authutils" "go.uber.org/zap" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/descriptorpb" "google.golang.org/protobuf/types/known/structpb" "math/big" + "slices" + "strings" "github.com/ethereum/go-ethereum/common/math" - "github.com/singnet/snet-daemon/v5/authutils" "github.com/singnet/snet-daemon/v5/utils" ) -// message used to sign is of the form ("__create_model", mpe_address, current_block_number) -func (ds *DaemonService) verifySignature(request *AuthorizationDetails) error { - return authutils.VerifySigner(ds.getMessageBytes(request.Message, request), - request.GetSignature(), utils.ToChecksumAddress(request.SignerAddress)) +var unifiedAuthMethods = map[string]struct{}{ + "validate_model_price": {}, + "train_model_price": {}, + "get_all_models": {}, + "get_model": {}, +} + +func (ds *DaemonService) verifySignature(r *AuthorizationDetails, method any) error { + fullMethodName, ok := method.(string) + if !ok { + return errors.New("invalid method") + } + + lastSlash := strings.LastIndex(fullMethodName, "/") + + methodName := fullMethodName[lastSlash+1:] + + _, isUnifiedMethod := unifiedAuthMethods[methodName] + + zap.L().Debug("Verifying signature", zap.String("methodName", methodName), zap.Bool("isUnifiedMethod", isUnifiedMethod), zap.String("msg", r.Message)) + + // good cases: + // methodName - get_model, msg - unified + // methodName - get_model, msg - get_model + // methodName - train_model, msg - train_model + + var allowDifference uint64 + + if strings.EqualFold(methodName, r.Message) { + allowDifference = 5 + } else if isUnifiedMethod && strings.EqualFold(r.Message, "unified") { + allowDifference = 600 + } else { + return fmt.Errorf("unsupported message: %s for this method", r.Message) + } + + if err := authutils.VerifySigner(ds.getMessageBytes(r.Message, r), r.GetSignature(), utils.ToChecksumAddress(r.SignerAddress)); err != nil { + return err + } + return ds.blockchain.CompareWithLatestBlockNumber(big.NewInt(0).SetUint64(r.CurrentBlock), allowDifference) } // "user passed message ", user_address, current_block_number @@ -65,15 +105,6 @@ func difference(oldAddresses []string, newAddresses []string) []string { return diff } -func isValuePresent(value string, list []string) bool { - for _, v := range list { - if v == value { - return true - } - } - return false -} - //go:embed training.proto var TrainingProtoEmbeded string @@ -176,3 +207,30 @@ func parseTrainingMetadata(protos linker.Files) (methodsMD map[string]*MethodMet zap.L().Debug("training methods", zap.Any("methods", trainingMD.TrainingMethods)) return } + +func paginate[T any](items []T, page, pageSize int) []T { + if page < 0 { + page = 0 + } + if pageSize < 1 { + pageSize = 1 + } + + start := page * pageSize + if start >= len(items) { + return []T{} + } + + end := start + pageSize + if end > len(items) { + end = len(items) + } + + return items[start:end] +} + +func sliceContainsEqualFold(slice []string, value string) bool { + return slices.ContainsFunc(slice, func(s string) bool { + return strings.EqualFold(s, value) + }) +} diff --git a/training/util_test.go b/training/util_test.go new file mode 100644 index 00000000..b1fa7346 --- /dev/null +++ b/training/util_test.go @@ -0,0 +1,146 @@ +package training + +import ( + "reflect" + "testing" +) + +func TestPaginate(t *testing.T) { + data := []any{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"} + + tests := []struct { + name string + page int + pageSize int + expected []any + }{ + {"First page", 0, 3, []any{"A", "B", "C"}}, + {"Second page", 1, 3, []any{"D", "E", "F"}}, + {"Last full page", 2, 3, []any{"G", "H", "I"}}, + {"Last partial page", 3, 3, []any{"J"}}, + {"Out of bounds", 4, 3, []any{}}, + {"Page size larger than data", 0, 15, data}, + {"Negative page", -1, 3, []any{"A", "B", "C"}}, + {"Zero page size", 0, 0, []any{"A"}}, + {"Negative page size", 0, -5, []any{"A"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := paginate(data, tt.page, tt.pageSize) + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("paginate(%d, %d) = %v, expected %v", tt.page, tt.pageSize, result, tt.expected) + } + }) + } +} + +func TestDifference(t *testing.T) { + tests := []struct { + name string + oldAddresses []string + newAddresses []string + expected []string + }{ + { + name: "Removed element", + oldAddresses: []string{"a", "blockchain", "c"}, + newAddresses: []string{"a", "c"}, + expected: []string{"blockchain"}, + }, + { + name: "Added element", + oldAddresses: []string{"a", "blockchain"}, + newAddresses: []string{"a", "blockchain", "c"}, + expected: []string{"c"}, + }, + { + name: "Removed and added elements", + oldAddresses: []string{"a", "blockchain", "c"}, + newAddresses: []string{"blockchain", "d"}, + expected: []string{"a", "c", "d"}, + }, + { + name: "No changes", + oldAddresses: []string{"a", "blockchain", "c"}, + newAddresses: []string{"a", "blockchain", "c"}, + expected: nil, + }, + { + name: "Both lists empty", + oldAddresses: []string{}, + newAddresses: []string{}, + expected: nil, + }, + { + name: "Old list empty", + oldAddresses: []string{}, + newAddresses: []string{"a", "blockchain"}, + expected: []string{"a", "blockchain"}, + }, + { + name: "New list empty", + oldAddresses: []string{"a", "blockchain"}, + newAddresses: []string{}, + expected: []string{"a", "blockchain"}, + }, + { + name: "Duplicate", + oldAddresses: []string{"a", "blockchain"}, + newAddresses: []string{"a", "a", "blockchain", "c"}, + expected: []string{"c"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := difference(tt.oldAddresses, tt.newAddresses) + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, result) + } + }) + } +} + +func TestSliceContainsEqualFold(t *testing.T) { + tests := []struct { + name string + slice []string + value string + expect bool + }{ + { + name: "Address found (same case)", + slice: []string{"0xabc", "0xdef", "0x123"}, + value: "0xabc", + expect: true, + }, + { + name: "Address found (different case)", + slice: []string{"0xabc", "0xdef", "0x123"}, + value: "0xABC", // should match "0xabc" because of EqualFold + expect: true, + }, + { + name: "Address not found", + slice: []string{"0xabc", "0xdef", "0x123"}, + value: "0x456", // not present + expect: false, + }, + { + name: "Empty slice", + slice: []string{}, + value: "0xabc", // no elements to match + expect: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := sliceContainsEqualFold(tt.slice, tt.value) + if got != tt.expect { + t.Errorf("sliceContainsEqualFold() = %v, want %v", got, tt.expect) + } + }) + } +} From 67921dfd54bd7f87944ab1078266e361437f497d Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Thu, 20 Feb 2025 10:01:32 +0300 Subject: [PATCH 17/22] optimize eth clients & refactor --- .github/workflows/test.yml | 2 +- authutils/auth_service.go | 19 +----- authutils/auth_service_test.go | 19 +----- blockchain/blockchain.go | 40 ++++++++---- blockchain/blockchain_test.go | 64 +++++++++++++++++++ .../{ethereumClient.go => ethereum_client.go} | 12 +--- ...orginzationMetadata.go => org_metadata.go} | 0 ...nMetadata_test.go => org_metadata_test.go} | 0 ...serviceMetadata.go => service_metadata.go} | 7 +- ...adata_test.go => service_metadata_test.go} | 6 +- config/blockchain_network_config.go | 4 +- config/blockchain_network_config_test.go | 3 +- config/config.go | 2 +- .../listen_organization_metadata_changing.go | 6 ++ go.mod | 12 ++-- go.sum | 14 ++++ ipfsutils/compressed.go | 4 +- metrics/utils.go | 1 - ratelimit/{rateLimit.go => rate_limit.go} | 0 .../{rateLimit_test.go => rate_limit_test.go} | 0 token/{jwttoken.go => jwt.go} | 0 token/{jwttoken_test.go => jwt_test.go} | 0 22 files changed, 138 insertions(+), 77 deletions(-) create mode 100644 blockchain/blockchain_test.go rename blockchain/{ethereumClient.go => ethereum_client.go} (88%) rename blockchain/{orginzationMetadata.go => org_metadata.go} (100%) rename blockchain/{orginzationMetadata_test.go => org_metadata_test.go} (100%) rename blockchain/{serviceMetadata.go => service_metadata.go} (98%) rename blockchain/{serviceMetadata_test.go => service_metadata_test.go} (98%) rename ratelimit/{rateLimit.go => rate_limit.go} (100%) rename ratelimit/{rateLimit_test.go => rate_limit_test.go} (100%) rename token/{jwttoken.go => jwt.go} (100%) rename token/{jwttoken_test.go => jwt_test.go} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ddddee0..9a0fcc05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: - name: install protoc (protobuf) uses: arduino/setup-protoc@v3 with: - version: "27.2" + version: "29.3" include-pre-releases: false - name: chmod to allow run script diff --git a/authutils/auth_service.go b/authutils/auth_service.go index 581f0978..e19583cf 100755 --- a/authutils/auth_service.go +++ b/authutils/auth_service.go @@ -71,8 +71,8 @@ func GetSignerAddressFromMessage(message, signature []byte) (signer *common.Addr keyOwnerAddressFieldLog := zap.Any("keyOwnerAddress", keyOwnerAddress) zap.L().Debug("Message signature parsed", //messageFieldLog, - signatureFieldLog, - messageHashFieldLog, + //signatureFieldLog, + //messageHashFieldLog, //publicKeyFieldLog, keyOwnerAddressFieldLog) @@ -106,22 +106,9 @@ func CompareWithLatestBlockNumber(blockNumberPassed *big.Int) error { return nil } -// CheckIfTokenHasExpired Check if the block number ( date on which the token was issued is not more than 1 month) -func CheckIfTokenHasExpired(expiredBlock *big.Int) error { - currentBlockNumber, err := CurrentBlock() - if err != nil { - return err - } - - if expiredBlock.Cmp(currentBlockNumber) < 0 { - return fmt.Errorf("authentication failed as the Free Call Token passed has expired") - } - return nil -} - // CurrentBlock Get the current block number from on chain func CurrentBlock() (*big.Int, error) { - if ethHttpClient, _, err := blockchain.CreateEthereumClients(); err != nil { + if ethHttpClient, err := blockchain.CreateHTTPEthereumClient(); err != nil { return nil, err } else { defer ethHttpClient.RawClient.Close() diff --git a/authutils/auth_service_test.go b/authutils/auth_service_test.go index 21a494ce..2ed89e53 100644 --- a/authutils/auth_service_test.go +++ b/authutils/auth_service_test.go @@ -3,12 +3,10 @@ package authutils import ( "github.com/ethereum/go-ethereum/common" - "math/big" - "testing" - "time" - "github.com/singnet/snet-daemon/v5/config" "github.com/stretchr/testify/assert" + "math/big" + "testing" ) func TestCompareWithLatestBlockNumber(t *testing.T) { @@ -21,19 +19,6 @@ func TestCompareWithLatestBlockNumber(t *testing.T) { currentBlockNum, _ = CurrentBlock() err = CompareWithLatestBlockNumber(currentBlockNum.Add(currentBlockNum, big.NewInt(1))) assert.Equal(t, nil, err) - -} - -func TestCheckAllowedBlockDifferenceForToken(t *testing.T) { - config.Vip().Set(config.BlockChainNetworkSelected, "sepolia") - config.Validate() - currentBlockNum, _ := CurrentBlock() - err := CheckIfTokenHasExpired(currentBlockNum.Sub(currentBlockNum, big.NewInt(20000))) - assert.Equal(t, err.Error(), "authentication failed as the Free Call Token passed has expired") - time.Sleep(250 * time.Millisecond) // because of HTTP 429 Too Many Requests - currentBlockNum, _ = CurrentBlock() - err = CheckIfTokenHasExpired(currentBlockNum.Add(currentBlockNum, big.NewInt(20))) - assert.Equal(t, nil, err) } func TestVerifyAddress(t *testing.T) { diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index 5ce85480..ac1c82a6 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -56,18 +56,14 @@ func NewProcessor(metadata *ServiceMetadata) (Processor, error) { } // Setup ethereum client - - if ethHttpClients, ethWSClients, err := CreateEthereumClients(); err != nil { + if ethHttpClients, err := CreateEthereumClient(); err != nil { return p, errors.Wrap(err, "error creating RPC client") } else { p.rawHttpClient = ethHttpClients.RawClient p.ethHttpClient = ethHttpClients.EthClient - p.rawWSClient = ethWSClients.RawClient - p.ethWSClient = ethWSClients.EthClient } // TODO: if address is not in config, try to load it using network - //TODO: Read this from github p.escrowContractAddress = metadata.GetMpeAddress() @@ -92,6 +88,13 @@ func (processor *Processor) ReconnectToWsClient() error { zap.L().Debug("Try to reconnect to websocket client") + return processor.ConnectToWsClient() +} + +func (processor *Processor) ConnectToWsClient() error { + + zap.L().Debug("Try to connect to websocket client") + newEthWSClients, err := CreateWSEthereumClient() if err != nil { return err @@ -124,17 +127,24 @@ func (processor *Processor) GetEthWSClient() *ethclient.Client { } func (processor *Processor) CurrentBlock() (currentBlock *big.Int, err error) { - // We have to do a raw call because the standard method of ethClient.HeaderByNumber(ctx, nil) errors on - // unmarshaling the response currently. See https://github.com/ethereum/go-ethereum/issues/3230 - var currentBlockHex string - if err = processor.rawHttpClient.CallContext(context.Background(), ¤tBlockHex, "eth_blockNumber"); err != nil { + latestBlock, err := processor.ethHttpClient.BlockNumber(context.Background()) + if err != nil { zap.L().Error("error determining current block", zap.Error(err)) return nil, fmt.Errorf("error determining current block: %v", err) } + return new(big.Int).SetUint64(latestBlock), nil +} - currentBlockBytes := common.FromHex(currentBlockHex) - currentBlock = new(big.Int).SetBytes(currentBlockBytes) +func (processor *Processor) CompareWithLatestBlockNumber(blockNumberPassed *big.Int, allowedBlockChainDifference uint64) (err error) { + latestBlockNumber, err := processor.CurrentBlock() + if err != nil { + return err + } + differenceInBlockNumber := blockNumberPassed.Sub(blockNumberPassed, latestBlockNumber) + if differenceInBlockNumber.Abs(differenceInBlockNumber).Uint64() > allowedBlockChainDifference { + return fmt.Errorf("authentication failed as the signature passed has expired") + } return } @@ -145,6 +155,10 @@ func (processor *Processor) HasIdentity() bool { func (processor *Processor) Close() { processor.ethHttpClient.Close() processor.rawHttpClient.Close() - processor.ethWSClient.Close() - processor.rawWSClient.Close() + if processor.ethWSClient != nil { + processor.ethWSClient.Close() + } + if processor.rawWSClient != nil { + processor.rawWSClient.Close() + } } diff --git a/blockchain/blockchain_test.go b/blockchain/blockchain_test.go new file mode 100644 index 00000000..70c9f94a --- /dev/null +++ b/blockchain/blockchain_test.go @@ -0,0 +1,64 @@ +package blockchain + +import ( + "github.com/singnet/snet-daemon/v5/config" + "math/big" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +const metadataJson = "{\n \"version\": 1,\n \"display_name\": \"semyon_dev\",\n \"encoding\": \"proto\",\n \"service_type\": \"grpc\",\n \"service_api_source\": \"ipfs://QmV9bBsLAZfXGibdU3isPwDn8SxAPkqg6YzcKamSCxnBCR\",\n \"mpe_address\": \"0x7E0aF8988DF45B824b2E0e0A87c6196897744970\",\n \"groups\": [\n {\n \"group_name\": \"default_group\",\n \"endpoints\": [\n \"http://localhost:7000\"\n ],\n \"pricing\": [\n {\n \"price_model\": \"fixed_price\",\n \"price_in_cogs\": 1,\n \"default\": true\n }\n ],\n \"group_id\": \"FtNuizEOUsVCd5f2Fij9soehtRSb58LlTePgkVnsgVI=\",\n \"free_call_signer_address\": \"0x747155e03c892B8b311B7Cfbb920664E8c6792fA\",\n \"free_calls\": 25,\n \"daemon_addresses\": [\n \"0x747155e03c892B8b311B7Cfbb920664E8c6792fA\"\n ]\n },\n {\n \"group_name\": \"not_default\",\n \"endpoints\": [\n \"http://localhost:7000\"\n ],\n \"pricing\": [\n {\n \"price_model\": \"fixed_price\",\n \"price_in_cogs\": 1,\n \"default\": true\n }\n ],\n \"group_id\": \"udN0SLIvsDdvQQe3Ltv/NwqCh7sPKdz4scYmlI7AMdE=\",\n \"free_call_signer_address\": \"0x747155e03c892B8b311B7Cfbb920664E8c6792fA\",\n \"free_calls\": 35,\n \"daemon_addresses\": [\n \"0x747155e03c892B8b311B7Cfbb920664E8c6792fA\"\n ]\n }\n ],\n \"assets\": {},\n \"media\": [],\n \"tags\": [],\n \"service_description\": {\n \"description\": \"Test service with localhost endpoint!\",\n \"url\": \"\"\n }\n}" + +// ProcessorTestSuite is a test suite for the Processor struct +type ProcessorTestSuite struct { + suite.Suite + processor *Processor +} + +// SetupSuite initializes the Ethereum client before running the tests +func (suite *ProcessorTestSuite) SetupSuite() { + config.Vip().Set(config.BlockchainEnabledKey, true) + config.Vip().Set(config.BlockChainNetworkSelected, "sepolia") + config.Validate() + srv, err := InitServiceMetaDataFromJson([]byte(metadataJson)) + if err != nil { + return + } + p, err := NewProcessor(srv) + assert.Nil(suite.T(), err) + suite.processor = &p +} + +// ✅ Test: If the block number difference is within the allowed limit → no error +func (suite *ProcessorTestSuite) TestCompareWithLatestBlockNumber_WithinLimit() { + latestBlock, err := suite.processor.CurrentBlock() + suite.Require().NoError(err, "CurrentBlock() should not return an error") + + // Simulate a block number within the allowed range (+2) + blockNumberPassed := new(big.Int).Add(latestBlock, big.NewInt(2)) + err = suite.processor.CompareWithLatestBlockNumber(blockNumberPassed, 5) + + // Expect no error + assert.NoError(suite.T(), err, "Expected no error when block difference is within the limit") +} + +// ❌ Test: If the block number difference exceeds the allowed limit → return an error +func (suite *ProcessorTestSuite) TestCompareWithLatestBlockNumber_ExceedsLimit() { + latestBlock, err := suite.processor.CurrentBlock() + suite.Require().NoError(err, "CurrentBlock() should not return an error") + + // Simulate a block number exceeding the allowed limit (+10) + blockNumberPassed := new(big.Int).Add(latestBlock, big.NewInt(10)) + err = suite.processor.CompareWithLatestBlockNumber(blockNumberPassed, 5) + + // Expect an error + assert.Error(suite.T(), err, "Expected an error when block difference exceeds the limit") + assert.Contains(suite.T(), err.Error(), "authentication failed", "Error message should indicate signature expiration") +} + +// Run the test suite +func TestProcessorTestSuite(t *testing.T) { + suite.Run(t, new(ProcessorTestSuite)) +} diff --git a/blockchain/ethereumClient.go b/blockchain/ethereum_client.go similarity index 88% rename from blockchain/ethereumClient.go rename to blockchain/ethereum_client.go index d0d797fb..d81dc9f7 100644 --- a/blockchain/ethereumClient.go +++ b/blockchain/ethereum_client.go @@ -22,18 +22,12 @@ func basicAuth(username, password string) string { return base64.StdEncoding.EncodeToString([]byte(auth)) } -func CreateEthereumClients() (*EthereumClient, *EthereumClient, error) { +func CreateEthereumClient() (*EthereumClient, error) { ethereumHttpClient, err := CreateHTTPEthereumClient() if err != nil { - return nil, nil, err + return nil, err } - - ethereumWsClient, err := CreateWSEthereumClient() - if err != nil { - return nil, nil, err - } - - return ethereumHttpClient, ethereumWsClient, nil + return ethereumHttpClient, nil } func CreateHTTPEthereumClient() (*EthereumClient, error) { diff --git a/blockchain/orginzationMetadata.go b/blockchain/org_metadata.go similarity index 100% rename from blockchain/orginzationMetadata.go rename to blockchain/org_metadata.go diff --git a/blockchain/orginzationMetadata_test.go b/blockchain/org_metadata_test.go similarity index 100% rename from blockchain/orginzationMetadata_test.go rename to blockchain/org_metadata_test.go diff --git a/blockchain/serviceMetadata.go b/blockchain/service_metadata.go similarity index 98% rename from blockchain/serviceMetadata.go rename to blockchain/service_metadata.go index 8517365e..86266ae4 100644 --- a/blockchain/serviceMetadata.go +++ b/blockchain/service_metadata.go @@ -421,8 +421,8 @@ func setDefaultPricing(metaData *ServiceMetadata) (err error) { return nil } } - err = fmt.Errorf("MetaData does not have the default pricing set ") - zap.L().Warn("Error in set default pricing", zap.Error(err)) + err = fmt.Errorf("metadata does not have the default pricing set") + zap.L().Warn("[setDefaultPricing] Error in set default pricing", zap.Error(err)) return err } @@ -432,13 +432,12 @@ func setMultiPartyEscrowAddress(metaData *ServiceMetadata) { } func setFreeCallData(metaData *ServiceMetadata) error { - if metaData.defaultGroup.FreeCalls > 0 { metaData.isfreeCallAllowed = true metaData.freeCallsAllowed = metaData.defaultGroup.FreeCalls //If the signer address is not a valid address, then return back an error if !common.IsHexAddress(metaData.defaultGroup.FreeCallSigner) { - return fmt.Errorf("MetaData does not have 'free_call_signer_address defined correctly" + errs.ErrDescURL(errs.InvalidMetadata)) + return fmt.Errorf("metadata does not have 'free_call_signer_address defined correctly") } metaData.freeCallSignerAddress = common.HexToAddress(ToChecksumAddress(metaData.defaultGroup.FreeCallSigner)) } diff --git a/blockchain/serviceMetadata_test.go b/blockchain/service_metadata_test.go similarity index 98% rename from blockchain/serviceMetadata_test.go rename to blockchain/service_metadata_test.go index 3e8a9017..9b777270 100644 --- a/blockchain/serviceMetadata_test.go +++ b/blockchain/service_metadata_test.go @@ -69,11 +69,11 @@ func TestInitServiceMetaDataFromJson(t *testing.T) { //Parse Bad JSON _, err = InitServiceMetaDataFromJson([]byte(strings.Replace(testJsonData, "0x7DF35C98f41F3Af0df1dc4c7F7D4C19a71Dd059F", "", 1))) if err != nil { - assert.Contains(t, err.Error(), "MetaData does not have 'free_call_signer_address defined correctly") + assert.Contains(t, err.Error(), "metadata does not have 'free_call_signer_address defined correctly") } _, err = InitServiceMetaDataFromJson([]byte(strings.Replace(testJsonData, "default_pricing", "dummy", 1))) if err != nil { - assert.Equal(t, err.Error(), "MetaData does not have the default pricing set ") + assert.Equal(t, err.Error(), "metadata does not have the default pricing set") } } @@ -98,7 +98,7 @@ func Test_setDefaultPricing(t *testing.T) { err := setDefaultPricing(&ServiceMetadata{}) assert.NotNil(t, err) err = setDefaultPricing(&ServiceMetadata{Groups: []OrganizationGroup{{GroupName: "default_group"}}}) - assert.Equal(t, err.Error(), "MetaData does not have the default pricing set ") + assert.Equal(t, err.Error(), "metadata does not have the default pricing set") } func Test_setGroup(t *testing.T) { diff --git a/config/blockchain_network_config.go b/config/blockchain_network_config.go index 619d9b3e..5d7abbc7 100644 --- a/config/blockchain_network_config.go +++ b/config/blockchain_network_config.go @@ -95,13 +95,13 @@ func setRegistryAddress() (err error) { // fileName, GetString(BlockChainNetworkSelected), err) //} - if err = deriveDatafromJSON(data); err != nil { + if err = deriveDataFromJSON(data); err != nil { return err } return nil } -func deriveDatafromJSON(data []byte) (err error) { +func deriveDataFromJSON(data []byte) (err error) { m := map[string]any{} err = json.Unmarshal(data, &m) if err != nil { diff --git a/config/blockchain_network_config_test.go b/config/blockchain_network_config_test.go index acaeff5e..4060fd76 100644 --- a/config/blockchain_network_config_test.go +++ b/config/blockchain_network_config_test.go @@ -117,7 +117,7 @@ func Test_setRegistryAddress(t *testing.T) { wantErr bool }{ {"11155111", false}, - {"5", false}, + {"1", false}, {"11155111_", true}, } @@ -130,7 +130,6 @@ func Test_setRegistryAddress(t *testing.T) { } }) } - } func Test_setBlockChainNetworkDetails(t *testing.T) { diff --git a/config/config.go b/config/config.go index 64d400ca..88ee9c6f 100644 --- a/config/config.go +++ b/config/config.go @@ -115,7 +115,7 @@ const ( "payment_channel_storage_client": { "connection_timeout": "0s", "request_timeout": "0s", - "hot_reload": true + "hot_reload": false }, "payment_channel_storage_server": { "id": "storage-1", diff --git a/contract_event_listener/listen_organization_metadata_changing.go b/contract_event_listener/listen_organization_metadata_changing.go index 40d5ed20..d50251db 100644 --- a/contract_event_listener/listen_organization_metadata_changing.go +++ b/contract_event_listener/listen_organization_metadata_changing.go @@ -21,6 +21,12 @@ func (l *ContractEventListener) ListenOrganizationMetadataChanging() { } ethWSClient := l.BlockchainProcessor.GetEthWSClient() + if ethWSClient == nil { + err := l.BlockchainProcessor.ConnectToWsClient() + if err != nil { + zap.L().Warn("[ListenOrganizationMetadataChanging]", zap.Error(err)) + } + } registryFilterer := blockchain.GetRegistryFilterer(ethWSClient) orgIdFilter := blockchain.MakeTopicFilterer(l.CurrentOrganizationMetaData.OrgID) diff --git a/go.mod b/go.mod index 43057742..d52a402e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/OneOfOne/go-utils v0.0.0-20180319162427-6019ff89a94e github.com/bufbuild/protocompile v0.14.1 github.com/emicklei/proto v1.14.0 - github.com/ethereum/go-ethereum v1.15.0 + github.com/ethereum/go-ethereum v1.15.2 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gorilla/handlers v1.5.2 @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/improbable-eng/grpc-web v0.15.0 github.com/ipfs/go-cid v0.5.0 - github.com/ipfs/kubo v0.33.1 + github.com/ipfs/kubo v0.33.2 github.com/magiconair/properties v1.8.9 github.com/pkg/errors v0.9.1 github.com/rs/cors v1.11.1 @@ -23,7 +23,7 @@ require ( github.com/singnet/snet-ecosystem-contracts v0.2.1 github.com/soheilhy/cmux v0.1.5 github.com/spf13/cast v1.7.1 - github.com/spf13/cobra v1.8.1 + github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 @@ -31,7 +31,7 @@ require ( go.etcd.io/etcd/server/v3 v3.5.18 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.33.0 - golang.org/x/net v0.34.0 + golang.org/x/net v0.35.0 golang.org/x/time v0.10.0 google.golang.org/grpc v1.70.0 google.golang.org/protobuf v1.36.5 @@ -62,7 +62,7 @@ require ( github.com/consensys/gnark-crypto v0.16.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect @@ -228,7 +228,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/supranational/blst v0.3.13 // indirect + github.com/supranational/blst v0.3.14 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect diff --git a/go.sum b/go.sum index 049a9d9b..f4b45493 100644 --- a/go.sum +++ b/go.sum @@ -150,6 +150,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= @@ -202,6 +204,10 @@ github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.15.0 h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv09/1ME4= github.com/ethereum/go-ethereum v1.15.0/go.mod h1:4q+4t48P2C03sjqGvTXix5lEOplf5dz4CTosbjt5tGs= +github.com/ethereum/go-ethereum v1.15.1 h1:ZR5hh6NXem4hNnhMIrdPFMTGHo6USTwWn47hbs6gRj4= +github.com/ethereum/go-ethereum v1.15.1/go.mod h1:wGQINJKEVUunCeoaA9C9qKMQ9GEOsEIunzzqTUO2F6Y= +github.com/ethereum/go-ethereum v1.15.2 h1:CcU13w1IXOo6FvS60JGCTVcAJ5Ik6RkWoVIvziiHdTU= +github.com/ethereum/go-ethereum v1.15.2/go.mod h1:wGQINJKEVUunCeoaA9C9qKMQ9GEOsEIunzzqTUO2F6Y= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= @@ -545,6 +551,8 @@ github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0z github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= github.com/ipfs/kubo v0.33.1 h1:dvc1o7j//9HAKjZg1d1ODSPVxmUn5YR7Ut5JPnR+H+Q= github.com/ipfs/kubo v0.33.1/go.mod h1:qc3gVRqjx5y9mPvsEz+CO5nc1mXvy4r9Etvqga2lnMo= +github.com/ipfs/kubo v0.33.2 h1:lHB3FhXk9yFjCiYNTNLqOjlUy7WWrIFQzVvOkJg0Ohg= +github.com/ipfs/kubo v0.33.2/go.mod h1:Ie+fLMp88GbFt1K3xjG6sxfzXwoAYlYRMM/DFHtYf8I= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.14.2 h1:9ERr7KXpCC7If0rChZLhYDlyr6Bes6yRKPJnCO3hdHY= @@ -1049,6 +1057,8 @@ github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cA github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -1081,6 +1091,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.14 h1:xNMoHRJOTwMn63ip6qoWJ2Ymgvj7E2b9jY2FAwY+qRo= +github.com/supranational/blst v0.3.14/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -1347,6 +1359,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/ipfsutils/compressed.go b/ipfsutils/compressed.go index e7dc89f2..b0b371e3 100644 --- a/ipfsutils/compressed.go +++ b/ipfsutils/compressed.go @@ -38,12 +38,12 @@ func ReadFilesCompressed(compressedFile []byte) (protos map[string]string, err e } protos[name] = string(data) default: - err = fmt.Errorf(fmt.Sprintf("%s : %c %s %s\n", + err = fmt.Errorf("%s : %c %s %s\n", "Unknown file Type ", header.Typeflag, "in file", name, - )) + ) zap.L().Error(err.Error()) return nil, err } diff --git a/metrics/utils.go b/metrics/utils.go index 3490b1b7..ef3f2f5a 100644 --- a/metrics/utils.go +++ b/metrics/utils.go @@ -124,7 +124,6 @@ func SignMessageForMetering(req *http.Request, commonStats *CommonStats) { req.Header.Set("X-Serviceid", commonStats.ServiceID) req.Header.Set("X-Currentblocknumber", currentBlock.String()) req.Header.Set("X-Signature", b64.StdEncoding.EncodeToString(signature)) - } func getPrivateKeyForMetering() (privateKey *ecdsa.PrivateKey, err error) { diff --git a/ratelimit/rateLimit.go b/ratelimit/rate_limit.go similarity index 100% rename from ratelimit/rateLimit.go rename to ratelimit/rate_limit.go diff --git a/ratelimit/rateLimit_test.go b/ratelimit/rate_limit_test.go similarity index 100% rename from ratelimit/rateLimit_test.go rename to ratelimit/rate_limit_test.go diff --git a/token/jwttoken.go b/token/jwt.go similarity index 100% rename from token/jwttoken.go rename to token/jwt.go diff --git a/token/jwttoken_test.go b/token/jwt_test.go similarity index 100% rename from token/jwttoken_test.go rename to token/jwt_test.go From fe19dfb2edb6f08e271aa75fb7e1e7bb845d212d Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Thu, 20 Feb 2025 13:11:52 +0300 Subject: [PATCH 18/22] fix tests --- training/tests/functional/service_test.go | 121 +++++++++++----------- 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/training/tests/functional/service_test.go b/training/tests/functional/service_test.go index 16807fea..6c18762d 100644 --- a/training/tests/functional/service_test.go +++ b/training/tests/functional/service_test.go @@ -6,6 +6,7 @@ import ( "crypto/ecdsa" "math/big" "slices" + "strings" "testing" "time" @@ -26,7 +27,8 @@ import ( type DaemonServiceSuite struct { suite.Suite - blockchain *blockchain.Processor + blockchain blockchain.Processor + currentBlock *big.Int modelStorage *training.ModelStorage userModelStorage *training.ModelUserStorage pendingModelStorage *training.PendingModelStorage @@ -54,9 +56,18 @@ var ( func (suite *DaemonServiceSuite) SetupSuite() { // setup test config once suite.setupTestConfig() + err := config.Validate() + assert.Nil(suite.T(), err) // init service metadata and organization metadata serviceMetadata, err := blockchain.InitServiceMetaDataFromJson([]byte(testJsonServiceData)) + suite.blockchain, err = blockchain.NewProcessor(serviceMetadata) + if err != nil { + zap.L().Fatal("can't connect to blockchain") + return + } + suite.currentBlock, _ = suite.blockchain.CurrentBlock() + orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) if err != nil { zap.L().Fatal("Error in initinalize organization metadata from json", zap.Error(err)) @@ -81,7 +92,7 @@ func (suite *DaemonServiceSuite) SetupTest() { suite.pendingModelStorage = pendingModelStorage suite.daemonService = training.NewTrainingService( - suite.blockchain, + &suite.blockchain, suite.serviceMetadata, suite.organizationMetadata, modelStorage, @@ -95,13 +106,13 @@ func (suite *DaemonServiceSuite) TearDownSuite() { suite.grpcServer.Stop() } -func getTestSignature(text string, blockNumber int, privateKey *ecdsa.PrivateKey) (signature []byte) { +func getTestSignature(text string, blockNumber uint64, privateKey *ecdsa.PrivateKey) (signature []byte) { HashPrefix32Bytes := []byte("\x19Ethereum Signed Message:\n32") message := bytes.Join([][]byte{ []byte(text), crypto.PubkeyToAddress(privateKey.PublicKey).Bytes(), - math.U256Bytes(big.NewInt(int64(blockNumber))), + math.U256Bytes(big.NewInt(0).SetUint64(blockNumber)), }, nil) hash := crypto.Keccak256( @@ -117,28 +128,28 @@ func getTestSignature(text string, blockNumber int, privateKey *ecdsa.PrivateKey return signature } -func createTestAuthDetails() *training.AuthorizationDetails { +func createTestAuthDetails(block *big.Int, method string) *training.AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { zap.L().Fatal("error in creating private key", zap.Error(err)) } return &training.AuthorizationDetails{ - CurrentBlock: 0, - Message: "__CreateModel", - Signature: getTestSignature("__CreateModel", 0, privateKey), + CurrentBlock: block.Uint64(), + Message: method, + Signature: getTestSignature(method, block.Uint64(), privateKey), SignerAddress: "0x3432cBa6BF635Df5fBFD1f1a794fA66D412b8774", } } -func creatBadTestAuthDetails() *training.AuthorizationDetails { +func creatBadTestAuthDetails(block *big.Int) *training.AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { zap.L().Fatal("error in creating private key", zap.Error(err)) } return &training.AuthorizationDetails{ - CurrentBlock: 0, + CurrentBlock: block.Uint64(), Message: "badMessage", - Signature: getTestSignature("badMessage", 0, privateKey), + Signature: getTestSignature("badMessage", block.Uint64(), privateKey), SignerAddress: "0x4444cBa6BF635Df5fBFD1f1a794fA66D412b8774", } } @@ -150,16 +161,17 @@ func (suite *DaemonServiceSuite) setupTestConfig() { "blockchain_network_selected": "sepolia", "daemon_end_point": "127.0.0.1:8080", "daemon_group_name":"default_group", - "payment_channel_storage_type": "etcd", + "payment_channel_storage_type": "_etcd", "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "ethereum_json_rpc_http_endpoint": "https://sepolia.infura.io/v3/09027f4a13e841d48dbfefc67e7685d5", "ipfs_timeout" : 30, "passthrough_enabled": true, - "passthrough_endpoint":"http://127.0.0.1:5002", + "passthrough_endpoint":"http://0.0.0.0:5001", + "model_maintenance_endpoint": "http://0.0.0.0:5001", + "model_training_endpoint": "http://0.0.0.0:5001", "service_id": "service_id", "organization_id": "test_org_id", "metering_enabled": false, - "ssl_cert": "", - "ssl_key": "", "max_message_size_in_mb" : 4, "daemon_type": "grpc", "enable_dynamic_pricing":false, @@ -175,39 +187,21 @@ func (suite *DaemonServiceSuite) setupTestConfig() { "timestamp_format": "2006-01-02T15:04:05.999Z07:00" }, "output": { - "type": ["file", "stdout"], - "file_pattern": "./snet-daemon.%Y%m%d.log", - "current_link": "./snet-daemon.log", - "max_size_in_mb": 10, - "max_age_in_days": 7, - "rotation_count": 0 + "type": ["stdout"] }, "hooks": [] }, - "model_maintenance_endpoint": "http://localhost:5001", "payment_channel_storage_client": { "connection_timeout": "0s", "request_timeout": "0s", "hot_reload": true }, "payment_channel_storage_server": { - "id": "storage-1", - "scheme": "http", - "host" : "127.0.0.1", - "client_port": 2379, - "peer_port": 2380, - "token": "unique-token", - "cluster": "storage-1=http://127.0.0.1:2380", - "startup_timeout": "1m", - "data_dir": "storage-data-dir-1.etcd", - "log_level": "info", - "log_outputs": ["./etcd-server.log"], "enabled": false }, "alerts_email": "", "service_heartbeat_type": "http", - "token_expiry_in_minutes": 1440, - "model_training_enabled": false + "model_training_enabled": true }` var testConfig = viper.New() @@ -374,7 +368,7 @@ func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, aut Authorization: authDetails, Model: newModel, } - response, err := suite.daemonService.CreateModel(context.Background(), request) + response, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request) if err != nil { zap.L().Fatal("error in creating additional test model", zap.Error(err)) } @@ -383,11 +377,11 @@ func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, aut } func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { - testAuthCreads := createTestAuthDetails() - badTestAuthCreads := creatBadTestAuthDetails() + testAuthCreads := createTestAuthDetails(suite.currentBlock, "get_model") + badTestAuthCreads := creatBadTestAuthDetails(suite.currentBlock) // check without request - response1, err := suite.daemonService.GetModel(context.Background(), nil) + response1, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), nil) assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) @@ -396,7 +390,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { Authorization: nil, ModelId: "test_2", } - response2, err := suite.daemonService.GetModel(context.Background(), request2) + response2, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request2) assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) assert.Equal(suite.T(), training.Status_ERRORED, response2.Status) @@ -405,7 +399,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { Authorization: badTestAuthCreads, ModelId: "test_2", } - response3, err := suite.daemonService.GetModel(context.Background(), request3) + response3, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request3) assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) @@ -414,7 +408,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { Authorization: testAuthCreads, ModelId: "", } - response4, err := suite.daemonService.GetModel(context.Background(), request4) + response4, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request4) assert.NotNil(suite.T(), err) assert.ErrorContains(suite.T(), err, training.ErrEmptyModelID.Error()) assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) @@ -424,7 +418,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { Authorization: testAuthCreads, ModelId: "test_2", } - response5, err := suite.daemonService.GetModel(context.Background(), request5) + response5, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request5) assert.ErrorContains(suite.T(), err, training.ErrAccessToModel.Error()) assert.Equal(suite.T(), &training.ModelResponse{}, response5) @@ -433,7 +427,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { Authorization: testAuthCreads, ModelId: "test_1", } - response6, err := suite.daemonService.GetModel(context.Background(), request6) + response6, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request6) assert.Nil(suite.T(), err) assert.NotEmpty(suite.T(), response6) assert.Equal(suite.T(), true, response6.IsPublic) @@ -443,15 +437,18 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { Authorization: testAuthCreads, ModelId: "test_3", } - response7, err := suite.daemonService.GetModel(context.Background(), request7) + response7, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request7) assert.Nil(suite.T(), err) assert.NotEmpty(suite.T(), response7) assert.Equal(suite.T(), false, response7.IsPublic) } func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { - testAuthCreads := createTestAuthDetails() - badTestAuthCreads := creatBadTestAuthDetails() + var err error + suite.currentBlock, err = suite.blockchain.CurrentBlock() + assert.Nil(suite.T(), err) + testAuthCreads := createTestAuthDetails(suite.currentBlock, "create_model") + badTestAuthCreads := creatBadTestAuthDetails(suite.currentBlock) newModel := &training.NewModel{ Name: "new_test_model", @@ -468,7 +465,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { newEmptyModel := &training.NewModel{} // check without request - response1, err := suite.daemonService.CreateModel(context.Background(), nil) + response1, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), nil) assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) @@ -486,16 +483,20 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { Authorization: badTestAuthCreads, Model: newModel, } - response3, err := suite.daemonService.CreateModel(context.Background(), request3) + response3, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request3) assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) + suite.currentBlock, err = suite.blockchain.CurrentBlock() + assert.Nil(suite.T(), err) + testAuthCreads = createTestAuthDetails(suite.currentBlock, "create_model") + // check with emptyModel request4 := &training.NewModelRequest{ Authorization: testAuthCreads, Model: newEmptyModel, } - response4, err := suite.daemonService.CreateModel(context.Background(), request4) + response4, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request4) assert.ErrorContains(suite.T(), err, training.ErrNoGRPCServiceOrMethod.Error()) assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) @@ -504,7 +505,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { Authorization: testAuthCreads, Model: newModel, } - response5, err := suite.daemonService.CreateModel(context.Background(), request5) + response5, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request5) assert.Nil(suite.T(), err) assert.Equal(suite.T(), training.Status_CREATED, response5.Status) @@ -527,7 +528,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - UserAddress: testUserAddress, + UserAddress: strings.ToLower(testUserAddress), } userModelData, ok, err := suite.userModelStorage.Get(userModelKey) @@ -537,15 +538,15 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { } func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { - testAuthCreads := createTestAuthDetails() - badTestAuthCreads := creatBadTestAuthDetails() + testAuthCreads := createTestAuthDetails(suite.currentBlock, "unified") + badTestAuthCreads := creatBadTestAuthDetails(suite.currentBlock) newAdditionalTestModelId := suite.createAdditionalTestModel("new_additional_test_model", testAuthCreads) expectedModelIds := []string{"test_3", newAdditionalTestModelId, "test_1"} // check without request - response1, err := suite.daemonService.GetAllModels(context.Background(), nil) + response1, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), nil) assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) assert.Nil(suite.T(), response1.ListOfModels) @@ -553,7 +554,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { request2 := &training.AllModelsRequest{ Authorization: nil, } - response2, err := suite.daemonService.GetAllModels(context.Background(), request2) + response2, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), request2) assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) assert.Nil(suite.T(), response2.ListOfModels) @@ -561,7 +562,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { request3 := &training.AllModelsRequest{ Authorization: badTestAuthCreads, } - response3, err := suite.daemonService.GetAllModels(context.Background(), request3) + response3, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), request3) assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) assert.Nil(suite.T(), response3.ListOfModels) @@ -569,7 +570,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { request4 := &training.AllModelsRequest{ Authorization: testAuthCreads, } - response4, err := suite.daemonService.GetAllModels(context.Background(), request4) + response4, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), request4) assert.Nil(suite.T(), err) modelIds := []string{} for _, model := range response4.ListOfModels { @@ -579,8 +580,8 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { assert.True(suite.T(), slices.Equal(expectedModelIds, modelIds)) } -func (suite *DaemonServiceSuite) TestDaemonSerice_ManageUpdateStatusWorkers() { - duration := time.Second * 10 +func (suite *DaemonServiceSuite) TestDaemonService_ManageUpdateStatusWorkers() { + duration := time.Second * 12 deadline := time.Now().Add(duration) ctx, cancel := context.WithDeadline(context.Background(), deadline) defer cancel() From e203f3f544a07c5f36e504c8e31539ae4bb7e67d Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Thu, 20 Feb 2025 13:39:14 +0300 Subject: [PATCH 19/22] fix tests --- .../test_provider_service.go | 4 +- .../{tests/functional => }/service_test.go | 196 +++++++++--------- 2 files changed, 100 insertions(+), 100 deletions(-) rename training/{tests/functional => service_mock}/test_provider_service.go (97%) rename training/{tests/functional => }/service_test.go (79%) diff --git a/training/tests/functional/test_provider_service.go b/training/service_mock/test_provider_service.go similarity index 97% rename from training/tests/functional/test_provider_service.go rename to training/service_mock/test_provider_service.go index 62ecd664..4dbfc6d7 100644 --- a/training/tests/functional/test_provider_service.go +++ b/training/service_mock/test_provider_service.go @@ -1,4 +1,4 @@ -package tests +package service_mock import ( "context" @@ -24,7 +24,7 @@ type model struct { status training.Status } -func startTestService(address string) *grpc.Server { +func StartTestService(address string) *grpc.Server { lis, err := net.Listen("tcp", address) if err != nil { log.Fatalf("failed to listen: %v", err) diff --git a/training/tests/functional/service_test.go b/training/service_test.go similarity index 79% rename from training/tests/functional/service_test.go rename to training/service_test.go index 6c18762d..49f52891 100644 --- a/training/tests/functional/service_test.go +++ b/training/service_test.go @@ -1,9 +1,10 @@ -package tests +package training import ( "bytes" "context" "crypto/ecdsa" + "github.com/singnet/snet-daemon/v5/training/service_mock" "math/big" "slices" "strings" @@ -22,20 +23,19 @@ import ( "github.com/singnet/snet-daemon/v5/blockchain" "github.com/singnet/snet-daemon/v5/config" "github.com/singnet/snet-daemon/v5/storage" - "github.com/singnet/snet-daemon/v5/training" ) type DaemonServiceSuite struct { suite.Suite blockchain blockchain.Processor currentBlock *big.Int - modelStorage *training.ModelStorage - userModelStorage *training.ModelUserStorage - pendingModelStorage *training.PendingModelStorage - daemonService training.DaemonServer - unimplementedDaemonService training.DaemonServer - modelKeys []*training.ModelKey - pendingModelKeys []*training.ModelKey // using for checking updated status + modelStorage *ModelStorage + userModelStorage *ModelUserStorage + pendingModelStorage *PendingModelStorage + daemonService DaemonServer + unimplementedDaemonService DaemonServer + modelKeys []*ModelKey + pendingModelKeys []*ModelKey // using for checking updated status grpcServer *grpc.Server grpcClient *grpc.ClientConn serviceMetadata *blockchain.ServiceMetadata @@ -77,11 +77,11 @@ func (suite *DaemonServiceSuite) SetupSuite() { suite.organizationMetadata = orgMetadata // setup unimplemented daemon server once - suite.unimplementedDaemonService = training.NoTrainingDaemonServer{} + suite.unimplementedDaemonService = NoTrainingDaemonServer{} // setup test poriver service once address := "localhost:5001" - suite.grpcServer = startTestService(address) + suite.grpcServer = service_mock.StartTestService(address) } func (suite *DaemonServiceSuite) SetupTest() { @@ -91,7 +91,7 @@ func (suite *DaemonServiceSuite) SetupTest() { suite.userModelStorage = userModelStorage suite.pendingModelStorage = pendingModelStorage - suite.daemonService = training.NewTrainingService( + suite.daemonService = NewTrainingService( &suite.blockchain, suite.serviceMetadata, suite.organizationMetadata, @@ -128,12 +128,12 @@ func getTestSignature(text string, blockNumber uint64, privateKey *ecdsa.Private return signature } -func createTestAuthDetails(block *big.Int, method string) *training.AuthorizationDetails { +func createTestAuthDetails(block *big.Int, method string) *AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { zap.L().Fatal("error in creating private key", zap.Error(err)) } - return &training.AuthorizationDetails{ + return &AuthorizationDetails{ CurrentBlock: block.Uint64(), Message: method, Signature: getTestSignature(method, block.Uint64(), privateKey), @@ -141,12 +141,12 @@ func createTestAuthDetails(block *big.Int, method string) *training.Authorizatio } } -func creatBadTestAuthDetails(block *big.Int) *training.AuthorizationDetails { +func creatBadTestAuthDetails(block *big.Int) *AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { zap.L().Fatal("error in creating private key", zap.Error(err)) } - return &training.AuthorizationDetails{ + return &AuthorizationDetails{ CurrentBlock: block.Uint64(), Message: "badMessage", Signature: getTestSignature("badMessage", block.Uint64(), privateKey), @@ -213,18 +213,18 @@ func (suite *DaemonServiceSuite) setupTestConfig() { config.SetVip(testConfig) } -func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *training.ModelUserStorage, *training.PendingModelStorage, *training.PublicModelStorage) { +func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserStorage, *PendingModelStorage, *PublicModelStorage) { memStorage := storage.NewMemStorage() - modelStorage := training.NewModelStorage(memStorage, suite.organizationMetadata) - userModelStorage := training.NewUserModelStorage(memStorage, suite.organizationMetadata) - pendingModelStorage := training.NewPendingModelStorage(memStorage, suite.organizationMetadata) - publicModelStorage := training.NewPublicModelStorage(memStorage, suite.organizationMetadata) + modelStorage := NewModelStorage(memStorage, suite.organizationMetadata) + userModelStorage := NewUserModelStorage(memStorage, suite.organizationMetadata) + pendingModelStorage := NewPendingModelStorage(memStorage, suite.organizationMetadata) + publicModelStorage := NewPublicModelStorage(memStorage, suite.organizationMetadata) - modelA := &training.ModelData{ + modelA := &ModelData{ IsPublic: true, ModelName: "testModel", AuthorizedAddresses: []string{}, - Status: training.Status_VALIDATING, + Status: Status_VALIDATING, CreatedByAddress: "address", ModelId: "test_1", UpdatedByAddress: "string", @@ -238,18 +238,18 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr UpdatedDate: "string", } - modelAKey := &training.ModelKey{ + modelAKey := &ModelKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", ModelId: "test_1", } - modelB := &training.ModelData{ + modelB := &ModelData{ IsPublic: false, ModelName: "testModel", AuthorizedAddresses: []string{}, - Status: training.Status_CREATED, + Status: Status_CREATED, CreatedByAddress: "address", ModelId: "test_2", UpdatedByAddress: "string", @@ -263,18 +263,18 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr UpdatedDate: "string", } - modelBKey := &training.ModelKey{ + modelBKey := &ModelKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", ModelId: "test_2", } - modelC := &training.ModelData{ + modelC := &ModelData{ IsPublic: false, ModelName: "testModel", AuthorizedAddresses: []string{}, - Status: training.Status_CREATED, + Status: Status_CREATED, CreatedByAddress: "address", ModelId: "test_3", UpdatedByAddress: "string", @@ -288,7 +288,7 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr UpdatedDate: "string", } - modelCKey := &training.ModelKey{ + modelCKey := &ModelKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", @@ -300,14 +300,14 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr modelStorage.Put(modelCKey, modelC) // adding to user models sotrage - userModelKey := &training.ModelUserKey{ + userModelKey := &ModelUserKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", UserAddress: testUserAddress, } - userModelData := &training.ModelUserData{ + userModelData := &ModelUserData{ ModelIds: []string{"test_3"}, OrganizationId: "test_org_id", ServiceId: "service_id", @@ -318,41 +318,41 @@ func (suite *DaemonServiceSuite) createTestModels() (*training.ModelStorage, *tr userModelStorage.Put(userModelKey, userModelData) // adding to pending models storage - pendingModelKey := &training.PendingModelKey{ + pendingModelKey := &PendingModelKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", } - pendingModelsData := &training.PendingModelData{ + pendingModelsData := &PendingModelData{ ModelIDs: []string{"test_1"}, } pendingModelStorage.Put(pendingModelKey, pendingModelsData) // adding to public models storage - publicModelKey := &training.PublicModelKey{ + publicModelKey := &PublicModelKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", } - publicModelsData := &training.PublicModelData{ + publicModelsData := &PublicModelData{ ModelIDs: []string{"test_1"}, } publicModelStorage.Put(publicModelKey, publicModelsData) // setup keys in suite - suite.modelKeys = []*training.ModelKey{modelAKey, modelBKey, modelCKey} - suite.pendingModelKeys = []*training.ModelKey{modelAKey} + suite.modelKeys = []*ModelKey{modelAKey, modelBKey, modelCKey} + suite.pendingModelKeys = []*ModelKey{modelAKey} // return all model keys, storages return modelStorage, userModelStorage, pendingModelStorage, publicModelStorage } -func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, authDetails *training.AuthorizationDetails) string { - newModel := &training.NewModel{ +func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, authDetails *AuthorizationDetails) string { + newModel := &NewModel{ Name: modelName, Description: "test_desc", GrpcMethodName: "test_grpc_method_name", @@ -364,7 +364,7 @@ func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, aut GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", } - request := &training.NewModelRequest{ + request := &NewModelRequest{ Authorization: authDetails, Model: newModel, } @@ -382,48 +382,48 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { // check without request response1, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), nil) - assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) + assert.ErrorContains(suite.T(), err, ErrNoAuthorization.Error()) + assert.Equal(suite.T(), Status_ERRORED, response1.Status) // check without auth - request2 := &training.CommonRequest{ + request2 := &CommonRequest{ Authorization: nil, ModelId: "test_2", } response2, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request2) - assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response2.Status) + assert.ErrorContains(suite.T(), err, ErrNoAuthorization.Error()) + assert.Equal(suite.T(), Status_ERRORED, response2.Status) // check with bad auth - request3 := &training.CommonRequest{ + request3 := &CommonRequest{ Authorization: badTestAuthCreads, ModelId: "test_2", } response3, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request3) - assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) + assert.ErrorContains(suite.T(), err, ErrBadAuthorization.Error()) + assert.Equal(suite.T(), Status_ERRORED, response3.Status) // check modelId is not empty string - request4 := &training.CommonRequest{ + request4 := &CommonRequest{ Authorization: testAuthCreads, ModelId: "", } response4, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request4) assert.NotNil(suite.T(), err) - assert.ErrorContains(suite.T(), err, training.ErrEmptyModelID.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) + assert.ErrorContains(suite.T(), err, ErrEmptyModelID.Error()) + assert.Equal(suite.T(), Status_ERRORED, response4.Status) // check without access to model - request5 := &training.CommonRequest{ + request5 := &CommonRequest{ Authorization: testAuthCreads, ModelId: "test_2", } response5, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request5) - assert.ErrorContains(suite.T(), err, training.ErrAccessToModel.Error()) - assert.Equal(suite.T(), &training.ModelResponse{}, response5) + assert.ErrorContains(suite.T(), err, ErrAccessToModel.Error()) + assert.Equal(suite.T(), &ModelResponse{}, response5) // check access to public model - request6 := &training.CommonRequest{ + request6 := &CommonRequest{ Authorization: testAuthCreads, ModelId: "test_1", } @@ -433,7 +433,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { assert.Equal(suite.T(), true, response6.IsPublic) //check access to non public model - request7 := &training.CommonRequest{ + request7 := &CommonRequest{ Authorization: testAuthCreads, ModelId: "test_3", } @@ -450,7 +450,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { testAuthCreads := createTestAuthDetails(suite.currentBlock, "create_model") badTestAuthCreads := creatBadTestAuthDetails(suite.currentBlock) - newModel := &training.NewModel{ + newModel := &NewModel{ Name: "new_test_model", Description: "test_desc", GrpcMethodName: "test_grpc_method_name", @@ -462,55 +462,55 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", } - newEmptyModel := &training.NewModel{} + newEmptyModel := &NewModel{} // check without request response1, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), nil) - assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) + assert.ErrorContains(suite.T(), err, ErrNoAuthorization.Error()) + assert.Equal(suite.T(), Status_ERRORED, response1.Status) // check without auth - request2 := &training.NewModelRequest{ + request2 := &NewModelRequest{ Authorization: nil, Model: newModel, } response2, err := suite.daemonService.CreateModel(context.Background(), request2) - assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response2.Status) + assert.ErrorContains(suite.T(), err, ErrNoAuthorization.Error()) + assert.Equal(suite.T(), Status_ERRORED, response2.Status) // check with bad auth - request3 := &training.NewModelRequest{ + request3 := &NewModelRequest{ Authorization: badTestAuthCreads, Model: newModel, } response3, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request3) - assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) + assert.ErrorContains(suite.T(), err, ErrBadAuthorization.Error()) + assert.Equal(suite.T(), Status_ERRORED, response3.Status) suite.currentBlock, err = suite.blockchain.CurrentBlock() assert.Nil(suite.T(), err) testAuthCreads = createTestAuthDetails(suite.currentBlock, "create_model") // check with emptyModel - request4 := &training.NewModelRequest{ + request4 := &NewModelRequest{ Authorization: testAuthCreads, Model: newEmptyModel, } response4, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request4) - assert.ErrorContains(suite.T(), err, training.ErrNoGRPCServiceOrMethod.Error()) - assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) + assert.ErrorContains(suite.T(), err, ErrNoGRPCServiceOrMethod.Error()) + assert.Equal(suite.T(), Status_ERRORED, response4.Status) // check with auth - request5 := &training.NewModelRequest{ + request5 := &NewModelRequest{ Authorization: testAuthCreads, Model: newModel, } response5, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request5) assert.Nil(suite.T(), err) - assert.Equal(suite.T(), training.Status_CREATED, response5.Status) + assert.Equal(suite.T(), Status_CREATED, response5.Status) // check model creation in model storage - modelKey := &training.ModelKey{ + modelKey := &ModelKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", @@ -524,7 +524,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { assert.Equal(suite.T(), newModel.Name, modelData.ModelName) // check user model data creation in user model storage - userModelKey := &training.ModelUserKey{ + userModelKey := &ModelUserKey{ OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", @@ -547,27 +547,27 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { // check without request response1, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), nil) - assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.ErrorContains(suite.T(), err, ErrNoAuthorization.Error()) assert.Nil(suite.T(), response1.ListOfModels) // check without auth - request2 := &training.AllModelsRequest{ + request2 := &AllModelsRequest{ Authorization: nil, } response2, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), request2) - assert.ErrorContains(suite.T(), err, training.ErrNoAuthorization.Error()) + assert.ErrorContains(suite.T(), err, ErrNoAuthorization.Error()) assert.Nil(suite.T(), response2.ListOfModels) // check with bad auth - request3 := &training.AllModelsRequest{ + request3 := &AllModelsRequest{ Authorization: badTestAuthCreads, } response3, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), request3) - assert.ErrorContains(suite.T(), err, training.ErrBadAuthorization.Error()) + assert.ErrorContains(suite.T(), err, ErrBadAuthorization.Error()) assert.Nil(suite.T(), response3.ListOfModels) // check with auth and without filters - request4 := &training.AllModelsRequest{ + request4 := &AllModelsRequest{ Authorization: testAuthCreads, } response4, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), request4) @@ -597,48 +597,48 @@ func (suite *DaemonServiceSuite) TestDaemonService_ManageUpdateStatusWorkers() { modelData, ok, err := suite.modelStorage.Get(modelKey) assert.Nil(suite.T(), err) assert.True(suite.T(), ok) - assert.Equal(suite.T(), training.Status_VALIDATED, modelData.Status) + assert.Equal(suite.T(), Status_VALIDATED, modelData.Status) } } func (suite *DaemonServiceSuite) TestDaemonService_UnimplementedDaemonService() { - response1, err := suite.unimplementedDaemonService.CreateModel(context.TODO(), &training.NewModelRequest{}) + response1, err := suite.unimplementedDaemonService.CreateModel(context.TODO(), &NewModelRequest{}) assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), training.Status_ERRORED, response1.Status) + assert.Equal(suite.T(), Status_ERRORED, response1.Status) - _, err = suite.unimplementedDaemonService.ValidateModelPrice(context.TODO(), &training.AuthValidateRequest{}) + _, err = suite.unimplementedDaemonService.ValidateModelPrice(context.TODO(), &AuthValidateRequest{}) assert.NotNil(suite.T(), err) - response2, err := suite.unimplementedDaemonService.ValidateModel(context.TODO(), &training.AuthValidateRequest{}) + response2, err := suite.unimplementedDaemonService.ValidateModel(context.TODO(), &AuthValidateRequest{}) assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), training.Status_ERRORED, response2.Status) + assert.Equal(suite.T(), Status_ERRORED, response2.Status) - _, err = suite.unimplementedDaemonService.TrainModelPrice(context.TODO(), &training.CommonRequest{}) + _, err = suite.unimplementedDaemonService.TrainModelPrice(context.TODO(), &CommonRequest{}) assert.NotNil(suite.T(), err) - response3, err := suite.unimplementedDaemonService.TrainModel(context.TODO(), &training.CommonRequest{}) + response3, err := suite.unimplementedDaemonService.TrainModel(context.TODO(), &CommonRequest{}) assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), training.Status_ERRORED, response3.Status) + assert.Equal(suite.T(), Status_ERRORED, response3.Status) - response4, err := suite.unimplementedDaemonService.DeleteModel(context.TODO(), &training.CommonRequest{}) + response4, err := suite.unimplementedDaemonService.DeleteModel(context.TODO(), &CommonRequest{}) assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), training.Status_ERRORED, response4.Status) + assert.Equal(suite.T(), Status_ERRORED, response4.Status) - response5, err := suite.unimplementedDaemonService.GetAllModels(context.TODO(), &training.AllModelsRequest{}) + response5, err := suite.unimplementedDaemonService.GetAllModels(context.TODO(), &AllModelsRequest{}) assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), []*training.ModelResponse{}, response5.ListOfModels) + assert.Equal(suite.T(), []*ModelResponse{}, response5.ListOfModels) - response6, err := suite.unimplementedDaemonService.GetModel(context.TODO(), &training.CommonRequest{}) + response6, err := suite.unimplementedDaemonService.GetModel(context.TODO(), &CommonRequest{}) assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), training.Status_ERRORED, response6.Status) + assert.Equal(suite.T(), Status_ERRORED, response6.Status) - response7, err := suite.unimplementedDaemonService.UpdateModel(context.Background(), &training.UpdateModelRequest{}) + response7, err := suite.unimplementedDaemonService.UpdateModel(context.Background(), &UpdateModelRequest{}) assert.NotNil(suite.T(), err) - assert.Equal(suite.T(), training.Status_ERRORED, response7.Status) + assert.Equal(suite.T(), Status_ERRORED, response7.Status) _, err = suite.unimplementedDaemonService.GetTrainingMetadata(context.Background(), &emptypb.Empty{}) assert.NotNil(suite.T(), err) - _, err = suite.unimplementedDaemonService.GetMethodMetadata(context.TODO(), &training.MethodMetadataRequest{}) + _, err = suite.unimplementedDaemonService.GetMethodMetadata(context.TODO(), &MethodMetadataRequest{}) assert.NotNil(suite.T(), err) } From 4cf3ae911303d50c15a8aafc66bb46537e49d8d8 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Thu, 20 Feb 2025 14:04:44 +0300 Subject: [PATCH 20/22] fix get_all_models --- handler/unary_interceptor.go | 5 +- training/service.go | 7 +- .../service_mock/test_provider_service.go | 120 ------------------ training/service_test.go | 3 +- training/test_provider_service.go | 119 +++++++++++++++++ 5 files changed, 125 insertions(+), 129 deletions(-) delete mode 100644 training/service_mock/test_provider_service.go create mode 100644 training/test_provider_service.go diff --git a/handler/unary_interceptor.go b/handler/unary_interceptor.go index 4db744ad..7fd22c5b 100644 --- a/handler/unary_interceptor.go +++ b/handler/unary_interceptor.go @@ -42,8 +42,11 @@ func (interceptor *paymentValidationUnaryInterceptor) unaryIntercept(ctx context ctx = context.WithValue(ctx, "method", info.FullMethod) + lastSlash := strings.LastIndex(info.FullMethod, "/") + methodName := info.FullMethod[lastSlash+1:] + // pass non training requests and free requests - if !strings.Contains(info.FullMethod, "validate_model") && !strings.Contains(info.FullMethod, "train_model") { + if methodName != "validate_model" && methodName != "train_model" { resp, e := handler(ctx, req) if e != nil { zap.L().Warn("gRPC handler returned error", zap.Error(e)) diff --git a/training/service.go b/training/service.go index edab529f..6e5dc7c1 100644 --- a/training/service.go +++ b/training/service.go @@ -868,12 +868,7 @@ func (ds *DaemonService) GetAllModels(ctx context.Context, request *AllModelsReq if request.IsPublic == nil || !*request.IsPublic { - userModelKey := &ModelUserKey{ - OrganizationId: config.GetString(config.OrganizationId), - ServiceId: config.GetString(config.ServiceId), - GroupId: ds.organizationMetaData.GetGroupIdString(), - UserAddress: request.Authorization.SignerAddress, - } + userModelKey := ds.userStorage.buildModelUserKey(request.Authorization.SignerAddress) if data, ok, err := ds.userStorage.Get(userModelKey); data != nil && ok && err == nil { modelKey := &ModelKey{ diff --git a/training/service_mock/test_provider_service.go b/training/service_mock/test_provider_service.go deleted file mode 100644 index 4dbfc6d7..00000000 --- a/training/service_mock/test_provider_service.go +++ /dev/null @@ -1,120 +0,0 @@ -package service_mock - -import ( - "context" - "fmt" - "log" - "net" - - "github.com/singnet/snet-daemon/v5/training" - "go.uber.org/zap" - "google.golang.org/grpc" -) - -type model struct { - modelId string - name string - desc string - grpcMethodName string - grpcServiceName string - addressList []string - isPublic bool - serviceId string - groupId string - status training.Status -} - -func StartTestService(address string) *grpc.Server { - lis, err := net.Listen("tcp", address) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - - grpcServer := grpc.NewServer() - var trainingServer TrainServer - training.RegisterModelServer(grpcServer, &trainingServer) - - trainingServer.curModelId = 0 - - go func() { - zap.L().Info("Starting test service", zap.String("address", address)) - if err := grpcServer.Serve(lis); err != nil { - zap.L().Fatal("Error in starting grpcServer", zap.Error(err)) - } - }() - - return grpcServer -} - -type TrainServer struct { - training.UnimplementedModelServer - curModelId int - models []model -} - -func (s *TrainServer) CreateModel(ctx context.Context, newModel *training.NewModel) (*training.ModelID, error) { - modelIdStr := fmt.Sprintf("%v", s.curModelId) - createdModel := &model{ - modelId: modelIdStr, - name: newModel.Name, - desc: newModel.Description, - grpcMethodName: newModel.GrpcMethodName, - grpcServiceName: newModel.GrpcServiceName, - addressList: newModel.AddressList, - isPublic: newModel.IsPublic, - serviceId: newModel.ServiceId, - groupId: newModel.GroupId, - status: training.Status_CREATED, - } - s.models = append(s.models, *createdModel) - - s.curModelId += 1 - - return &training.ModelID{ - ModelId: fmt.Sprintf("%v", s.curModelId), - }, nil -} - -func (s *TrainServer) ValidateModelPrice(ctx context.Context, request *training.ValidateRequest) (*training.PriceInBaseUnit, error) { - return &training.PriceInBaseUnit{ - Price: 1, - }, nil -} - -func (s *TrainServer) UploadAndValidate(server training.Model_UploadAndValidateServer) error { - panic("implement me") -} - -func (s *TrainServer) ValidateModel(ctx context.Context, request *training.ValidateRequest) (*training.StatusResponse, error) { - return &training.StatusResponse{ - Status: training.Status_VALIDATING, - }, nil -} - -func (s *TrainServer) TrainModelPrice(ctx context.Context, id *training.ModelID) (*training.PriceInBaseUnit, error) { - return &training.PriceInBaseUnit{ - Price: 1, - }, nil -} - -func (s *TrainServer) TrainModel(ctx context.Context, id *training.ModelID) (*training.StatusResponse, error) { - return &training.StatusResponse{ - Status: training.Status_TRAINING, - }, nil -} - -func (s *TrainServer) DeleteModel(ctx context.Context, id *training.ModelID) (*training.StatusResponse, error) { - return &training.StatusResponse{ - Status: training.Status_DELETED, - }, nil -} - -func (s *TrainServer) GetModelStatus(ctx context.Context, id *training.ModelID) (*training.StatusResponse, error) { - return &training.StatusResponse{ - Status: training.Status_VALIDATED, - }, nil -} - -func (s *TrainServer) mustEmbedUnimplementedModelServer() { - panic("implement me") -} diff --git a/training/service_test.go b/training/service_test.go index 49f52891..9dbe3bbc 100644 --- a/training/service_test.go +++ b/training/service_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/ecdsa" - "github.com/singnet/snet-daemon/v5/training/service_mock" "math/big" "slices" "strings" @@ -81,7 +80,7 @@ func (suite *DaemonServiceSuite) SetupSuite() { // setup test poriver service once address := "localhost:5001" - suite.grpcServer = service_mock.StartTestService(address) + suite.grpcServer = startTestService(address) } func (suite *DaemonServiceSuite) SetupTest() { diff --git a/training/test_provider_service.go b/training/test_provider_service.go new file mode 100644 index 00000000..01cb3bbe --- /dev/null +++ b/training/test_provider_service.go @@ -0,0 +1,119 @@ +package training + +import ( + "context" + "fmt" + "log" + "net" + + "go.uber.org/zap" + "google.golang.org/grpc" +) + +type model struct { + modelId string + name string + desc string + grpcMethodName string + grpcServiceName string + addressList []string + isPublic bool + serviceId string + groupId string + status Status +} + +func startTestService(address string) *grpc.Server { + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + grpcServer := grpc.NewServer() + var trainingServer TestTrainServer + RegisterModelServer(grpcServer, &trainingServer) + + trainingServer.curModelId = 0 + + go func() { + zap.L().Info("Starting test service", zap.String("address", address)) + if err := grpcServer.Serve(lis); err != nil { + zap.L().Fatal("Error in starting grpcServer", zap.Error(err)) + } + }() + + return grpcServer +} + +type TestTrainServer struct { + UnimplementedModelServer + curModelId int + models []model +} + +func (s *TestTrainServer) CreateModel(ctx context.Context, newModel *NewModel) (*ModelID, error) { + modelIdStr := fmt.Sprintf("%v", s.curModelId) + createdModel := &model{ + modelId: modelIdStr, + name: newModel.Name, + desc: newModel.Description, + grpcMethodName: newModel.GrpcMethodName, + grpcServiceName: newModel.GrpcServiceName, + addressList: newModel.AddressList, + isPublic: newModel.IsPublic, + serviceId: newModel.ServiceId, + groupId: newModel.GroupId, + status: Status_CREATED, + } + s.models = append(s.models, *createdModel) + + s.curModelId += 1 + + return &ModelID{ + ModelId: fmt.Sprintf("%v", s.curModelId), + }, nil +} + +func (s *TestTrainServer) ValidateModelPrice(ctx context.Context, request *ValidateRequest) (*PriceInBaseUnit, error) { + return &PriceInBaseUnit{ + Price: 1, + }, nil +} + +func (s *TestTrainServer) UploadAndValidate(server Model_UploadAndValidateServer) error { + panic("implement me") +} + +func (s *TestTrainServer) ValidateModel(ctx context.Context, request *ValidateRequest) (*StatusResponse, error) { + return &StatusResponse{ + Status: Status_VALIDATING, + }, nil +} + +func (s *TestTrainServer) TrainModelPrice(ctx context.Context, id *ModelID) (*PriceInBaseUnit, error) { + return &PriceInBaseUnit{ + Price: 1, + }, nil +} + +func (s *TestTrainServer) TrainModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return &StatusResponse{ + Status: Status_TRAINING, + }, nil +} + +func (s *TestTrainServer) DeleteModel(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return &StatusResponse{ + Status: Status_DELETED, + }, nil +} + +func (s *TestTrainServer) GetModelStatus(ctx context.Context, id *ModelID) (*StatusResponse, error) { + return &StatusResponse{ + Status: Status_VALIDATED, + }, nil +} + +func (s *TestTrainServer) mustEmbedUnimplementedModelServer() { + panic("implement me") +} From 6f75e4a6b86c973488bc2acb272b2aaa35a66899 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Thu, 20 Feb 2025 20:43:14 +0300 Subject: [PATCH 21/22] fix service_test.go --- training/service_test.go | 54 ++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/training/service_test.go b/training/service_test.go index 9dbe3bbc..700eb195 100644 --- a/training/service_test.go +++ b/training/service_test.go @@ -15,7 +15,6 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/emptypb" @@ -62,14 +61,13 @@ func (suite *DaemonServiceSuite) SetupSuite() { serviceMetadata, err := blockchain.InitServiceMetaDataFromJson([]byte(testJsonServiceData)) suite.blockchain, err = blockchain.NewProcessor(serviceMetadata) if err != nil { - zap.L().Fatal("can't connect to blockchain") - return + suite.T().Fatalf("can't connect to blockchain: %v", err) } - suite.currentBlock, _ = suite.blockchain.CurrentBlock() + suite.currentBlock, err = suite.blockchain.CurrentBlock() orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) if err != nil { - zap.L().Fatal("Error in initinalize organization metadata from json", zap.Error(err)) + suite.T().Fatalf("Error in initinalize organization metadata from json: %v", err) } suite.serviceMetadata = serviceMetadata @@ -121,7 +119,7 @@ func getTestSignature(text string, blockNumber uint64, privateKey *ecdsa.Private signature, err := crypto.Sign(hash, privateKey) if err != nil { - zap.L().Fatal("Cannot sign test message", zap.Error(err)) + return nil } return signature @@ -130,7 +128,7 @@ func getTestSignature(text string, blockNumber uint64, privateKey *ecdsa.Private func createTestAuthDetails(block *big.Int, method string) *AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { - zap.L().Fatal("error in creating private key", zap.Error(err)) + return nil } return &AuthorizationDetails{ CurrentBlock: block.Uint64(), @@ -143,7 +141,7 @@ func createTestAuthDetails(block *big.Int, method string) *AuthorizationDetails func creatBadTestAuthDetails(block *big.Int) *AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { - zap.L().Fatal("error in creating private key", zap.Error(err)) + return nil } return &AuthorizationDetails{ CurrentBlock: block.Uint64(), @@ -206,7 +204,7 @@ func (suite *DaemonServiceSuite) setupTestConfig() { var testConfig = viper.New() err := config.ReadConfigFromJsonString(testConfig, testConfigJson) if err != nil { - zap.L().Fatal("Error in reading config") + suite.T().Fatalf("Error in reading config") } config.SetVip(testConfig) @@ -294,9 +292,18 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt ModelId: "test_3", } - modelStorage.Put(modelAKey, modelA) - modelStorage.Put(modelBKey, modelB) - modelStorage.Put(modelCKey, modelC) + err := modelStorage.Put(modelAKey, modelA) + if err != nil { + suite.T().Fatalf("error in putting model: %v", err) + } + err = modelStorage.Put(modelBKey, modelB) + if err != nil { + suite.T().Fatalf("error in putting model: %v", err) + } + err = modelStorage.Put(modelCKey, modelC) + if err != nil { + suite.T().Fatalf("error in putting model: %v", err) + } // adding to user models sotrage userModelKey := &ModelUserKey{ @@ -307,14 +314,15 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt } userModelData := &ModelUserData{ - ModelIds: []string{"test_3"}, + ModelIds: []string{"test_1", "test_2", "test_3"}, OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", UserAddress: testUserAddress, } - userModelStorage.Put(userModelKey, userModelData) + err = userModelStorage.Put(userModelKey, userModelData) + assert.Nil(suite.T(), err) // adding to pending models storage pendingModelKey := &PendingModelKey{ @@ -327,7 +335,8 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt ModelIDs: []string{"test_1"}, } - pendingModelStorage.Put(pendingModelKey, pendingModelsData) + err = pendingModelStorage.Put(pendingModelKey, pendingModelsData) + assert.Nil(suite.T(), err) // adding to public models storage publicModelKey := &PublicModelKey{ @@ -340,7 +349,8 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt ModelIDs: []string{"test_1"}, } - publicModelStorage.Put(publicModelKey, publicModelsData) + err = publicModelStorage.Put(publicModelKey, publicModelsData) + assert.Nil(suite.T(), err) // setup keys in suite suite.modelKeys = []*ModelKey{modelAKey, modelBKey, modelCKey} @@ -369,7 +379,7 @@ func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, aut } response, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request) if err != nil { - zap.L().Fatal("error in creating additional test model", zap.Error(err)) + suite.T().Fatalf("error in creating additional test model: %v", err) } return response.ModelId @@ -485,9 +495,8 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { response3, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request3) assert.ErrorContains(suite.T(), err, ErrBadAuthorization.Error()) assert.Equal(suite.T(), Status_ERRORED, response3.Status) - - suite.currentBlock, err = suite.blockchain.CurrentBlock() assert.Nil(suite.T(), err) + testAuthCreads = createTestAuthDetails(suite.currentBlock, "create_model") // check with emptyModel @@ -538,9 +547,10 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { testAuthCreads := createTestAuthDetails(suite.currentBlock, "unified") + testAuthCreadsCreateModel := createTestAuthDetails(suite.currentBlock, "create_model") badTestAuthCreads := creatBadTestAuthDetails(suite.currentBlock) - newAdditionalTestModelId := suite.createAdditionalTestModel("new_additional_test_model", testAuthCreads) + newAdditionalTestModelId := suite.createAdditionalTestModel("new_additional_test_model", testAuthCreadsCreateModel) expectedModelIds := []string{"test_3", newAdditionalTestModelId, "test_1"} @@ -587,9 +597,9 @@ func (suite *DaemonServiceSuite) TestDaemonService_ManageUpdateStatusWorkers() { select { case <-ctx.Done(): - zap.L().Info("Context done", zap.Error(ctx.Err())) + suite.T().Logf("context done %v", ctx.Err()) case <-time.After(duration): - zap.L().Info("Operation timed out after", zap.Duration("duration", duration)) + suite.T().Logf("operation timed out after: %v", duration) } for _, modelKey := range suite.pendingModelKeys { From a8ab93eaf20be8c8213a02ba1024cf6469d57a29 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Fri, 21 Feb 2025 17:04:43 +0300 Subject: [PATCH 22/22] fix service_test.go --- go.mod | 2 - go.sum | 95 +-------------------------------- training/service.go | 4 +- training/service_test.go | 110 +++++++++++++++++++-------------------- 4 files changed, 58 insertions(+), 153 deletions(-) diff --git a/go.mod b/go.mod index d52a402e..32f15b6d 100644 --- a/go.mod +++ b/go.mod @@ -196,7 +196,6 @@ require ( github.com/pion/rtp v1.8.11 // indirect github.com/pion/sctp v1.8.35 // indirect github.com/pion/sdp/v3 v3.0.10 // indirect - github.com/pion/srtp/v2 v2.0.20 // indirect github.com/pion/srtp/v3 v3.0.4 // indirect github.com/pion/stun v0.6.1 // indirect github.com/pion/stun/v2 v2.0.0 // indirect @@ -205,7 +204,6 @@ require ( github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/turn/v2 v2.1.6 // indirect github.com/pion/turn/v4 v4.0.0 // indirect - github.com/pion/webrtc/v3 v3.3.5 // indirect github.com/pion/webrtc/v4 v4.0.8 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.0 // indirect diff --git a/go.sum b/go.sum index f4b45493..5c5eda0f 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.19.1 h1:mv2yVhy96D2CuskLPXnc58oJNMs5PCWjAZuyYU0p12M= -github.com/bits-and-blooms/bitset v1.19.1/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -85,14 +83,11 @@ github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBT github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/caddyserver/certmagic v0.21.6 h1:1th6GfprVfsAtFNOu4StNMF5IxK5XiaI0yZhAHlZFPE= -github.com/caddyserver/certmagic v0.21.6/go.mod h1:n1sCo7zV1Ez2j+89wrzDxo4N/T1Ws/Vx8u5NvuBFabw= github.com/caddyserver/certmagic v0.21.7 h1:66KJioPFJwttL43KYSWk7ErSmE6LfaJgCQuhm8Sg6fg= github.com/caddyserver/certmagic v0.21.7/go.mod h1:LCPG3WLxcnjVKl/xpjzM0gqh0knrKKKiO5WVttX2eEI= github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -102,7 +97,6 @@ github.com/ceramicnetwork/go-dag-jose v0.1.1 h1:7pObs22egc14vSS3AfCFfS1VmaL4lQUs github.com/ceramicnetwork/go-dag-jose v0.1.1/go.mod h1:8ptnYwY2Z2y/s5oJnNBn/UCxLg6CpramNJ2ZXF/5aNY= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -129,12 +123,8 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/consensys/bavard v0.1.24 h1:Lfe+bjYbpaoT7K5JTFoMi5wo9V4REGLvQQbHmatoN2I= -github.com/consensys/bavard v0.1.24/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/bavard v0.1.29 h1:fobxIYksIQ+ZSrTJUuQgu+HIJwclrAPcdXqd7H2hh1k= github.com/consensys/bavard v0.1.29/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= -github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= -github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= github.com/consensys/gnark-crypto v0.16.0 h1:8Dl4eYmUWK9WmlP1Bj6je688gBRJCJbT8Mw4KoTAawo= github.com/consensys/gnark-crypto v0.16.0/go.mod h1:Ke3j06ndtPTVvo++PhGNgvm+lgpLvzbcE2MqljY7diU= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= @@ -148,8 +138,6 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= @@ -202,10 +190,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy4WDs= github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.15.0 h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv09/1ME4= -github.com/ethereum/go-ethereum v1.15.0/go.mod h1:4q+4t48P2C03sjqGvTXix5lEOplf5dz4CTosbjt5tGs= -github.com/ethereum/go-ethereum v1.15.1 h1:ZR5hh6NXem4hNnhMIrdPFMTGHo6USTwWn47hbs6gRj4= -github.com/ethereum/go-ethereum v1.15.1/go.mod h1:wGQINJKEVUunCeoaA9C9qKMQ9GEOsEIunzzqTUO2F6Y= github.com/ethereum/go-ethereum v1.15.2 h1:CcU13w1IXOo6FvS60JGCTVcAJ5Ik6RkWoVIvziiHdTU= github.com/ethereum/go-ethereum v1.15.2/go.mod h1:wGQINJKEVUunCeoaA9C9qKMQ9GEOsEIunzzqTUO2F6Y= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= @@ -229,8 +213,6 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= @@ -284,7 +266,6 @@ github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -374,8 +355,6 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc= github.com/google/pprof v0.0.0-20250208200701-d0013a598941/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -416,8 +395,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -549,8 +526,6 @@ github.com/ipfs/go-unixfsnode v1.9.2 h1:0A12BYs4XOtDPJTMlwmNPlllDfqcc4yie4e919hc github.com/ipfs/go-unixfsnode v1.9.2/go.mod h1:v1nuMFHf4QTIhFUdPMvg1nQu7AqDLvIdwyvJ531Ot1U= github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= -github.com/ipfs/kubo v0.33.1 h1:dvc1o7j//9HAKjZg1d1ODSPVxmUn5YR7Ut5JPnR+H+Q= -github.com/ipfs/kubo v0.33.1/go.mod h1:qc3gVRqjx5y9mPvsEz+CO5nc1mXvy4r9Etvqga2lnMo= github.com/ipfs/kubo v0.33.2 h1:lHB3FhXk9yFjCiYNTNLqOjlUy7WWrIFQzVvOkJg0Ohg= github.com/ipfs/kubo v0.33.2/go.mod h1:Ie+fLMp88GbFt1K3xjG6sxfzXwoAYlYRMM/DFHtYf8I= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= @@ -563,8 +538,6 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= -github.com/ipshipyard/p2p-forge v0.3.0 h1:mdeFqiq8ljX149OCQvveV0vOlKeIt4PWkJjXVfux/GE= -github.com/ipshipyard/p2p-forge v0.3.0/go.mod h1:L0TJMzniMEDjX8G+RB201U2woHvASwbsujNVDNVivDo= github.com/ipshipyard/p2p-forge v0.3.1 h1:Vr0l6wzX4zL7l8+UHJlsRBNHmlHpP3c//NrCZeGj4KU= github.com/ipshipyard/p2p-forge v0.3.1/go.mod h1:XQAvFJeXGo4oiyVPXkC3cph//5kF785L5Pjd3/kWFWo= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -604,8 +577,6 @@ github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kK github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= -github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/koron/go-ssdp v0.0.5 h1:E1iSMxIs4WqxTbIBLtmNBeOOC+1sCIXQeqTWVnpmwhk= github.com/koron/go-ssdp v0.0.5/go.mod h1:Qm59B7hpKpDqfyRNWRNr00jGwLdXjDyZh6y7rH6VS0w= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -636,26 +607,18 @@ github.com/libp2p/go-doh-resolver v0.5.0 h1:4h7plVVW+XTS+oUBw2+8KfoM1jF6w8XmO7+s github.com/libp2p/go-doh-resolver v0.5.0/go.mod h1:aPDxfiD2hNURgd13+hfo29z9IC22fv30ee5iM31RzxU= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.38.2 h1:9SZQDOCi82A25An4kx30lEtr6kGTxrtoaDkbs5xrK5k= -github.com/libp2p/go-libp2p v0.38.2/go.mod h1:QWV4zGL3O9nXKdHirIC59DoRcZ446dfkjbOJ55NEWFo= github.com/libp2p/go-libp2p v0.39.0 h1:LmrhDRud4eDkQCSB4l5NfoIFDqvDwAyANmfeYkgnKgs= github.com/libp2p/go-libp2p v0.39.0/go.mod h1:3zicI8Lp7Isun+Afo/JOACUbbJqqR2owK6RQWFsVAbI= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= -github.com/libp2p/go-libp2p-kad-dht v0.28.2 h1:/VivUl/Ru0tVgkWNhDDBy8pK6q+gRdI+z8VfqmSUJWo= -github.com/libp2p/go-libp2p-kad-dht v0.28.2/go.mod h1:sUR/qh4p/5+YFXBtwOiCmIBeBA2YD94ttmL+Xk8+pTE= github.com/libp2p/go-libp2p-kad-dht v0.29.0 h1:045eW21lGlMSD9aKSZZGH4fnBMIInPwQLxIQ35P962I= github.com/libp2p/go-libp2p-kad-dht v0.29.0/go.mod h1:mIci3rHSwDsxQWcCjfmxD8vMTgh5xLuvwb1D5WP8ZNk= -github.com/libp2p/go-libp2p-kbucket v0.6.4 h1:OjfiYxU42TKQSB8t8WYd8MKhYhMJeO2If+NiuKfb6iQ= -github.com/libp2p/go-libp2p-kbucket v0.6.4/go.mod h1:jp6w82sczYaBsAypt5ayACcRJi0lgsba7o4TzJKEfWA= github.com/libp2p/go-libp2p-kbucket v0.6.5 h1:Fsl1YvZcMwqrR4DYrTO02yo9PGYs2HBQIT3lGXFMTxg= github.com/libp2p/go-libp2p-kbucket v0.6.5/go.mod h1:U6WOd0BvnSp03IQSrjgM54tg7zh1UUNsXLJqAQzClTA= github.com/libp2p/go-libp2p-pubsub v0.12.0 h1:PENNZjSfk8KYxANRlpipdS7+BfLmOl3L2E/6vSNjbdI= github.com/libp2p/go-libp2p-pubsub v0.12.0/go.mod h1:Oi0zw9aw8/Y5GC99zt+Ef2gYAl+0nZlwdJonDyOz/sE= github.com/libp2p/go-libp2p-pubsub-router v0.6.0 h1:D30iKdlqDt5ZmLEYhHELCMRj8b4sFAqrUcshIUvVP/s= github.com/libp2p/go-libp2p-pubsub-router v0.6.0/go.mod h1:FY/q0/RBTKsLA7l4vqC2cbRbOvyDotg8PJQ7j8FDudE= -github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= -github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.4 h1:6LqS1Bzn5CfDJ4tzvP9uwh42IB7TJLNFJA6dEeGBv84= @@ -672,9 +635,8 @@ github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFP github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= -github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= -github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/libp2p/go-yamux/v4 v4.0.2 h1:nrLh89LN/LEiqcFiqdKDRHjGstN300C1269K/EX0CPU= +github.com/libp2p/go-yamux/v4 v4.0.2/go.mod h1:C808cCRgOs1iBwY4S71T5oxgMxgLmqUw56qh4AeBW2o= github.com/libp2p/zeroconf/v2 v2.2.0 h1:Cup06Jv6u81HLhIj1KasuNM/RHHrJ8T7wOTS4+Tv53Q= github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -702,14 +664,10 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mholt/acmez/v3 v3.0.0 h1:r1NcjuWR0VaKP2BTjDK9LRFBw/WvURx3jlaEUl9Ht8E= -github.com/mholt/acmez/v3 v3.0.0/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/mholt/acmez/v3 v3.0.1 h1:4PcjKjaySlgXK857aTfDuRbmnM5gb3Ruz3tvoSJAUp8= github.com/mholt/acmez/v3 v3.0.1/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= -github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -801,8 +759,6 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= -github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -810,9 +766,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= -github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -834,8 +789,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -858,7 +811,6 @@ github.com/pion/ice/v4 v4.0.6 h1:jmM9HwI9lfetQV/39uD0nY4y++XZNPhvzIPCb8EwxUM= github.com/pion/ice/v4 v4.0.6/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI= github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y= -github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= @@ -868,22 +820,14 @@ github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= -github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.10 h1:puphjdbjPB+L+NFaVuZ5h6bt1g5q4kFIoI+r5q/g0CU= -github.com/pion/rtp v1.8.10/go.mod h1:8uMBJj32Pa1wwx8Fuv/AsFhn8jsgw+3rUC2PfoBZ8p4= github.com/pion/rtp v1.8.11 h1:17xjnY5WO5hgO6SD3/NTIUPvSFw/PbLsIJyz1r1yNIk= github.com/pion/rtp v1.8.11/go.mod h1:8uMBJj32Pa1wwx8Fuv/AsFhn8jsgw+3rUC2PfoBZ8p4= github.com/pion/sctp v1.8.35 h1:qwtKvNK1Wc5tHMIYgTDJhfZk7vATGVHhXbUDfHbYwzA= github.com/pion/sctp v1.8.35/go.mod h1:EcXP8zCYVTRy3W9xtOF7wJm1L1aXfKRQzaM33SjQlzg= -github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= -github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= github.com/pion/sdp/v3 v3.0.10 h1:6MChLE/1xYB+CjumMw+gZ9ufp2DPApuVSnDT8t5MIgA= github.com/pion/sdp/v3 v3.0.10/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= -github.com/pion/srtp/v2 v2.0.20 h1:HNNny4s+OUmG280ETrCdgFndp4ufx3/uy85EawYEhTk= -github.com/pion/srtp/v2 v2.0.20/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M= github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -893,7 +837,6 @@ github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLcc github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= @@ -905,8 +848,6 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM= github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA= -github.com/pion/webrtc/v3 v3.3.5 h1:ZsSzaMz/i9nblPdiAkZoP+E6Kmjw+jnyq3bEmU3EtRg= -github.com/pion/webrtc/v3 v3.3.5/go.mod h1:liNa+E1iwyzyXqNUwvoMRNQ10x8h8FOeJKL8RkIbamE= github.com/pion/webrtc/v4 v4.0.8 h1:T1ZmnT9qxIJIt4d8XoiMOBrTClGHDDXNg9e/fh018Qc= github.com/pion/webrtc/v4 v4.0.8/go.mod h1:HHBeUVBAC+j4ZFnYhovEFStF02Arb1EyD4G7e7HBTJw= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -983,14 +924,10 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= -github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -1021,8 +958,6 @@ github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go. github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= -github.com/singnet/snet-ecosystem-contracts v0.1.1 h1:d/xa8T6iFt9efLKATY9SNIu+/lL47Hn9NwszjIEKjnQ= -github.com/singnet/snet-ecosystem-contracts v0.1.1/go.mod h1:upRHFLALLPm2chI/tdYGuF/4Kh8RB6rjdVR8HnH27SI= github.com/singnet/snet-ecosystem-contracts v0.2.1 h1:7zX+b+y1bgCyMexaTSRT1zFvgS0iFYQQ3KlVNUYGs7k= github.com/singnet/snet-ecosystem-contracts v0.2.1/go.mod h1:upRHFLALLPm2chI/tdYGuF/4Kh8RB6rjdVR8HnH27SI= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1048,15 +983,11 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1089,8 +1020,6 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= -github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/supranational/blst v0.3.14 h1:xNMoHRJOTwMn63ip6qoWJ2Ymgvj7E2b9jY2FAwY+qRo= github.com/supranational/blst v0.3.14/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= @@ -1125,8 +1054,6 @@ github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboa github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= -github.com/whyrusleeping/cbor-gen v0.1.2 h1:WQFlrPhpcQl+M2/3dP5cvlTLWPVsL6LGBb9jJt6l/cA= -github.com/whyrusleeping/cbor-gen v0.1.2/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/cbor-gen v0.2.0 h1:v8DREoK/1qQBSc6/UZ4OgU06+9FkywTh8glX0Hi+jkc= github.com/whyrusleeping/cbor-gen v0.2.0/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= @@ -1154,8 +1081,6 @@ github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCR github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= -go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -1265,8 +1190,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1278,8 +1201,6 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34= golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1306,8 +1227,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1357,8 +1276,6 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1519,8 +1436,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= -gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -1564,16 +1479,10 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20250127172529-29210b9bc287 h1:WoUI1G0DQ648FKvSl756SKxHQR/bI+y4HyyIQfxMWI8= -google.golang.org/genproto v0.0.0-20250127172529-29210b9bc287/go.mod h1:wkQ2Aj/xvshAUDtO/JHvu9y+AaN9cqs28QuSVSHtZSY= google.golang.org/genproto v0.0.0-20250207221924-e9438ea467c6 h1:SSk8oMbcHFbMwftDvX4PHbkqss3RkEZUF+k1h9d/sns= google.golang.org/genproto v0.0.0-20250207221924-e9438ea467c6/go.mod h1:wkQ2Aj/xvshAUDtO/JHvu9y+AaN9cqs28QuSVSHtZSY= -google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287 h1:A2ni10G3UlplFrWdCDJTl7D7mJ7GSRm37S+PDimaKRw= -google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4= google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 h1:L9JNMl/plZH9wmzQUHleO/ZZDSN+9Gh41wPczNy+5Fk= google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250204164813-702378808489 h1:5bKytslY8ViY0Cj/ewmRtrWHW64bNF03cAatUUFCdFI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250204164813-702378808489/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 h1:2duwAxN2+k0xLNpjnHTXoMUgnv6VPSp5fiqTuwSxjmI= google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= diff --git a/training/service.go b/training/service.go index 6e5dc7c1..0a074f7c 100644 --- a/training/service.go +++ b/training/service.go @@ -916,10 +916,10 @@ func (ds *DaemonService) GetAllModels(ctx context.Context, request *AllModelsReq for _, v := range modelDetailsArray { //zap.L().Debug("[GetAllModels] model", zap.String("id", v.ModelId), zap.String("status", v.Status.String())) - if strings.Contains(v.Name, request.Name) && + if strings.Contains(strings.ToLower(v.Name), strings.ToLower(request.Name)) && strings.Contains(v.GrpcMethodName, request.GrpcMethodName) && strings.Contains(v.GrpcServiceName, request.GrpcServiceName) && - strings.Contains(v.CreatedByAddress, strings.ToLower(request.CreatedByAddress)) && + strings.Contains(strings.ToLower(v.CreatedByAddress), strings.ToLower(request.CreatedByAddress)) && slices.Contains(request.Statuses, v.Status) { filtered = append(filtered, v) } diff --git a/training/service_test.go b/training/service_test.go index 9dbe3bbc..17687f97 100644 --- a/training/service_test.go +++ b/training/service_test.go @@ -4,20 +4,18 @@ import ( "bytes" "context" "crypto/ecdsa" - "math/big" - "slices" - "strings" - "testing" - "time" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/emptypb" + "math/big" + "slices" + "strings" + "testing" + "time" "github.com/singnet/snet-daemon/v5/blockchain" "github.com/singnet/snet-daemon/v5/config" @@ -49,7 +47,7 @@ var ( testJsonOrgGroupData = "{ \"org_name\": \"organization_name\", \"org_id\": \"test_org_id\", \"groups\": [ { \"group_name\": \"default_group2\", \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } }, { \"group_name\": \"default_group\", \"license_server_endpoints\": [\"https://licensendpoint:8082\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"payment\": { \"payment_address\": \"0x671276c61943A35D5F230d076bDFd91B0c47bF09\", \"payment_expiration_threshold\": 40320, \"payment_channel_storage_type\": \"etcd\", \"payment_channel_storage_client\": { \"connection_timeout\": \"15s\", \"request_timeout\": \"13s\", \"endpoints\": [ \"http://127.0.0.1:2379\" ] } } } ] }" testJsonServiceData = "{ \"version\": 1, \"display_name\": \"Example1\", \"encoding\": \"grpc\", \"service_type\": \"grpc\", \"payment_expiration_threshold\": 40320, \"model_ipfs_hash\": \"Qmdiq8Hu6dYiwp712GtnbBxagyfYyvUY1HYqkH7iN76UCc\", " + " \"mpe_address\": \"0x7E6366Fbe3bdfCE3C906667911FC5237Cc96BD08\", \"groups\": [ { \"free_calls\": 12, \"free_call_signer_address\": \"0x7DF35C98f41F3Af0df1dc4c7F7D4C19a71Dd059F\", \"endpoints\": [\"http://34.344.33.1:2379\",\"http://34.344.33.1:2389\"], \"group_id\": \"88ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\",\"group_name\": \"default_group\", \"pricing\": [ { \"price_model\": \"fixed_price\", \"price_in_cogs\": 2 }, { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"default\":true, \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] }, { \"endpoints\": [\"http://97.344.33.1:2379\",\"http://67.344.33.1:2389\"], \"group_id\": \"99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=\", \"pricing\": [ { \"package_name\": \"example_service\", \"price_model\": \"fixed_price_per_method\", \"details\": [ { \"service_name\": \"Calculator\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 2 }, { \"method_name\": \"mul\", \"price_in_cogs\": 3 } ] }, { \"service_name\": \"Calculator2\", \"method_pricing\": [ { \"method_name\": \"add\", \"price_in_cogs\": 2 }, { \"method_name\": \"sub\", \"price_in_cogs\": 1 }, { \"method_name\": \"div\", \"price_in_cogs\": 3 }, { \"method_name\": \"mul\", \"price_in_cogs\": 2 } ] } ] }] } ] } " - testUserAddress = "0x3432cBa6BF635Df5fBFD1f1a794fA66D412b8774" + testUserAddress = strings.ToLower("0x3432cBa6BF635Df5fBFD1f1a794fA66D412b8774") ) func (suite *DaemonServiceSuite) SetupSuite() { @@ -62,14 +60,13 @@ func (suite *DaemonServiceSuite) SetupSuite() { serviceMetadata, err := blockchain.InitServiceMetaDataFromJson([]byte(testJsonServiceData)) suite.blockchain, err = blockchain.NewProcessor(serviceMetadata) if err != nil { - zap.L().Fatal("can't connect to blockchain") - return + suite.T().Fatalf("can't connect to blockchain: %v", err) } - suite.currentBlock, _ = suite.blockchain.CurrentBlock() + suite.currentBlock, err = suite.blockchain.CurrentBlock() orgMetadata, err := blockchain.InitOrganizationMetaDataFromJson([]byte(testJsonOrgGroupData)) if err != nil { - zap.L().Fatal("Error in initinalize organization metadata from json", zap.Error(err)) + suite.T().Fatalf("Error in initinalize organization metadata from json: %v", err) } suite.serviceMetadata = serviceMetadata @@ -121,7 +118,7 @@ func getTestSignature(text string, blockNumber uint64, privateKey *ecdsa.Private signature, err := crypto.Sign(hash, privateKey) if err != nil { - zap.L().Fatal("Cannot sign test message", zap.Error(err)) + return nil } return signature @@ -130,7 +127,7 @@ func getTestSignature(text string, blockNumber uint64, privateKey *ecdsa.Private func createTestAuthDetails(block *big.Int, method string) *AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { - zap.L().Fatal("error in creating private key", zap.Error(err)) + return nil } return &AuthorizationDetails{ CurrentBlock: block.Uint64(), @@ -143,7 +140,7 @@ func createTestAuthDetails(block *big.Int, method string) *AuthorizationDetails func creatBadTestAuthDetails(block *big.Int) *AuthorizationDetails { privateKey, err := crypto.HexToECDSA("c0e4803a3a5b3c26cfc96d19a6dc4bbb4ba653ce5fa68f0b7dbf3903cda17ee6") if err != nil { - zap.L().Fatal("error in creating private key", zap.Error(err)) + return nil } return &AuthorizationDetails{ CurrentBlock: block.Uint64(), @@ -162,7 +159,6 @@ func (suite *DaemonServiceSuite) setupTestConfig() { "daemon_group_name":"default_group", "payment_channel_storage_type": "_etcd", "ipfs_end_point": "http://ipfs.singularitynet.io:80", - "ethereum_json_rpc_http_endpoint": "https://sepolia.infura.io/v3/09027f4a13e841d48dbfefc67e7685d5", "ipfs_timeout" : 30, "passthrough_enabled": true, "passthrough_endpoint":"http://0.0.0.0:5001", @@ -206,7 +202,7 @@ func (suite *DaemonServiceSuite) setupTestConfig() { var testConfig = viper.New() err := config.ReadConfigFromJsonString(testConfig, testConfigJson) if err != nil { - zap.L().Fatal("Error in reading config") + suite.T().Fatalf("Error in reading config") } config.SetVip(testConfig) @@ -222,11 +218,11 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt modelA := &ModelData{ IsPublic: true, ModelName: "testModel", - AuthorizedAddresses: []string{}, + AuthorizedAddresses: []string{testUserAddress}, Status: Status_VALIDATING, - CreatedByAddress: "address", + CreatedByAddress: testUserAddress, ModelId: "test_1", - UpdatedByAddress: "string", + UpdatedByAddress: testUserAddress, GroupId: "string", OrganizationId: "string", ServiceId: "string", @@ -249,9 +245,9 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt ModelName: "testModel", AuthorizedAddresses: []string{}, Status: Status_CREATED, - CreatedByAddress: "address", - ModelId: "test_2", - UpdatedByAddress: "string", + CreatedByAddress: "unknown", + ModelId: "test_2_no_access", + UpdatedByAddress: "unknown", GroupId: "string", OrganizationId: "string", ServiceId: "string", @@ -266,17 +262,17 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - ModelId: "test_2", + ModelId: "test_2_no_access", } modelC := &ModelData{ IsPublic: false, ModelName: "testModel", - AuthorizedAddresses: []string{}, + AuthorizedAddresses: []string{testUserAddress}, Status: Status_CREATED, - CreatedByAddress: "address", + CreatedByAddress: testUserAddress, ModelId: "test_3", - UpdatedByAddress: "string", + UpdatedByAddress: testUserAddress, GroupId: "string", OrganizationId: "string", ServiceId: "string", @@ -294,9 +290,18 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt ModelId: "test_3", } - modelStorage.Put(modelAKey, modelA) - modelStorage.Put(modelBKey, modelB) - modelStorage.Put(modelCKey, modelC) + err := modelStorage.Put(modelAKey, modelA) + if err != nil { + suite.T().Fatalf("error in putting model: %v", err) + } + err = modelStorage.Put(modelBKey, modelB) + if err != nil { + suite.T().Fatalf("error in putting model: %v", err) + } + err = modelStorage.Put(modelCKey, modelC) + if err != nil { + suite.T().Fatalf("error in putting model: %v", err) + } // adding to user models sotrage userModelKey := &ModelUserKey{ @@ -307,14 +312,15 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt } userModelData := &ModelUserData{ - ModelIds: []string{"test_3"}, + ModelIds: []string{"test_1", "test_3"}, OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", UserAddress: testUserAddress, } - userModelStorage.Put(userModelKey, userModelData) + err = userModelStorage.Put(userModelKey, userModelData) + assert.Nil(suite.T(), err) // adding to pending models storage pendingModelKey := &PendingModelKey{ @@ -327,7 +333,8 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt ModelIDs: []string{"test_1"}, } - pendingModelStorage.Put(pendingModelKey, pendingModelsData) + err = pendingModelStorage.Put(pendingModelKey, pendingModelsData) + assert.Nil(suite.T(), err) // adding to public models storage publicModelKey := &PublicModelKey{ @@ -340,7 +347,8 @@ func (suite *DaemonServiceSuite) createTestModels() (*ModelStorage, *ModelUserSt ModelIDs: []string{"test_1"}, } - publicModelStorage.Put(publicModelKey, publicModelsData) + err = publicModelStorage.Put(publicModelKey, publicModelsData) + assert.Nil(suite.T(), err) // setup keys in suite suite.modelKeys = []*ModelKey{modelAKey, modelBKey, modelCKey} @@ -369,7 +377,7 @@ func (suite *DaemonServiceSuite) createAdditionalTestModel(modelName string, aut } response, err := suite.daemonService.CreateModel(context.WithValue(context.Background(), "method", "create_model"), request) if err != nil { - zap.L().Fatal("error in creating additional test model", zap.Error(err)) + suite.T().Fatalf("error in creating additional test model: %v", err) } return response.ModelId @@ -387,7 +395,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { // check without auth request2 := &CommonRequest{ Authorization: nil, - ModelId: "test_2", + ModelId: "test_2_no_access", } response2, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request2) assert.ErrorContains(suite.T(), err, ErrNoAuthorization.Error()) @@ -396,7 +404,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { // check with bad auth request3 := &CommonRequest{ Authorization: badTestAuthCreads, - ModelId: "test_2", + ModelId: "test_2_no_access", } response3, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request3) assert.ErrorContains(suite.T(), err, ErrBadAuthorization.Error()) @@ -415,7 +423,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetModel() { // check without access to model request5 := &CommonRequest{ Authorization: testAuthCreads, - ModelId: "test_2", + ModelId: "test_2_no_access", } response5, err := suite.daemonService.GetModel(context.WithValue(context.Background(), "method", "get_model"), request5) assert.ErrorContains(suite.T(), err, ErrAccessToModel.Error()) @@ -486,8 +494,6 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { assert.ErrorContains(suite.T(), err, ErrBadAuthorization.Error()) assert.Equal(suite.T(), Status_ERRORED, response3.Status) - suite.currentBlock, err = suite.blockchain.CurrentBlock() - assert.Nil(suite.T(), err) testAuthCreads = createTestAuthDetails(suite.currentBlock, "create_model") // check with emptyModel @@ -527,7 +533,7 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { OrganizationId: "test_org_id", ServiceId: "service_id", GroupId: "99ybRIg2wAx55mqVsA6sB4S7WxPQHNKqa4BPu/bhj+U=", - UserAddress: strings.ToLower(testUserAddress), + UserAddress: testUserAddress, } userModelData, ok, err := suite.userModelStorage.Get(userModelKey) @@ -538,11 +544,12 @@ func (suite *DaemonServiceSuite) TestDaemonService_CreateModel() { func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { testAuthCreads := createTestAuthDetails(suite.currentBlock, "unified") + testAuthCreadsCreateModel := createTestAuthDetails(suite.currentBlock, "create_model") badTestAuthCreads := creatBadTestAuthDetails(suite.currentBlock) - newAdditionalTestModelId := suite.createAdditionalTestModel("new_additional_test_model", testAuthCreads) + newAdditionalTestModelId := suite.createAdditionalTestModel("new_additional_test_model", testAuthCreadsCreateModel) - expectedModelIds := []string{"test_3", newAdditionalTestModelId, "test_1"} + expectedModelIds := []string{"test_1", "test_3", newAdditionalTestModelId} // check without request response1, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), nil) @@ -569,6 +576,8 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { request4 := &AllModelsRequest{ Authorization: testAuthCreads, } + + // from 0x3432cBa6BF635Df5fBFD1f1a794fA66D412b8774 response4, err := suite.daemonService.GetAllModels(context.WithValue(context.Background(), "method", "get_all_models"), request4) assert.Nil(suite.T(), err) modelIds := []string{} @@ -576,22 +585,11 @@ func (suite *DaemonServiceSuite) TestDaemonService_GetAllModels() { modelIds = append(modelIds, model.ModelId) } - assert.True(suite.T(), slices.Equal(expectedModelIds, modelIds)) + assert.ElementsMatch(suite.T(), expectedModelIds, modelIds) } func (suite *DaemonServiceSuite) TestDaemonService_ManageUpdateStatusWorkers() { - duration := time.Second * 12 - deadline := time.Now().Add(duration) - ctx, cancel := context.WithDeadline(context.Background(), deadline) - defer cancel() - - select { - case <-ctx.Done(): - zap.L().Info("Context done", zap.Error(ctx.Err())) - case <-time.After(duration): - zap.L().Info("Operation timed out after", zap.Duration("duration", duration)) - } - + time.Sleep(10 * time.Second) for _, modelKey := range suite.pendingModelKeys { modelData, ok, err := suite.modelStorage.Get(modelKey) assert.Nil(suite.T(), err)