diff --git a/docs/KeyManager.md b/docs/KeyManager.md new file mode 100644 index 000000000..a1a470814 --- /dev/null +++ b/docs/KeyManager.md @@ -0,0 +1,111 @@ +# KeyManager Architecture + +## Overview + +KeyManager supports multiple key source types through a provider abstraction pattern. This allows the system to support different key management solutions (raw private keys, GCP KMS, AWS KMS, etc.) without changing the core KeyManager logic. + +## Architecture + +``` +KeyManager + ├── IKeyProvider (interface) + │ ├── RawPrivateKeyProvider (implemented) + │ ├── GcpKmsProvider (future) + │ └── AwsKmsProvider (future) + │ + └── Factory Pattern + └── createKeyProvider(config) → IKeyProvider +``` + +## Components + +### 1. IKeyProvider Interface (`types.ts`) + +Base interface that all key providers must implement: + +```typescript +interface IKeyProvider { + getType(): KeyProviderType + initialize(): Promise + getPeerId(): PeerId + getLibp2pPrivateKey(): any + getLibp2pPublicKey(): Uint8Array + getEthAddress(): string + getRawPrivateKeyBytes(): Uint8Array + cleanup?(): Promise +} +``` + +### 2. KeyProviderType Enum + +Defines supported key provider types: + +- `RAW` - Raw private key from environment variable + +### 3. RawPrivateKeyProvider (`providers/RawPrivateKeyProvider.ts`) + +Implementation for raw private keys: + +- Loads private key from config +- Derives libp2p keys and peerId +- Derives Ethereum address +- Provides raw private key bytes for EVM signer creation + +### 4. Factory (`factory.ts`) + +Creates and initializes the appropriate key provider: + +```typescript +createKeyProvider(config: KeyProviderConfig): Promise +``` + +### 5. KeyManager (`index.ts`) + +Main class that: + +- Wraps a key provider +- Manages EVM signer caching +- Provides unified API for key access + +## Usage + +### Current Usage (Raw Private Key) + +```typescript +import { createKeyProvider } from './components/KeyManager/index.js' + +const keyManager = new KeyManager(config) +``` + +## Adding New Key Providers + +To add a new key provider (e.g., AWS KMS): + +1. **Add to KeyProviderType**: + +```typescript +export type KeyProviderType = 'raw' | 'gcp-kms' | 'aws' //new +``` + +2. **Create provider class**: + +```typescript +export class AwsKmsProvider implements IKeyProvider { + // Implement all interface methods +} +``` + +3. **Update factory**: + +```typescript +case 'aws'': + provider = new AwsKmsProvider(config) + break +``` + +## Benefits + +1. **Extensibility**: Easy to add new key sources +2. **Testability**: Can mock key providers for testing +3. **Security**: Supports secure key management solutions (KMS) +4. **Separation of Concerns**: Key retrieval logic separated from key usage logic diff --git a/src/@types/KeyManager.ts b/src/@types/KeyManager.ts new file mode 100644 index 000000000..d4d8246f5 --- /dev/null +++ b/src/@types/KeyManager.ts @@ -0,0 +1,94 @@ +import type { PeerId } from '@libp2p/interface' +import { Wallet } from 'ethers' +import { EncryptMethod } from './fileObject.js' +import { Readable } from 'stream' +/** + * Key provider types supported by KeyManager + */ + +export type KeyProviderType = 'raw' // 'gcp-kms' | 'aws-kms' in future +/** + * Base interface for key providers. + * Each key provider implementation must implement this interface. + * Initialization happens in the constructor. + */ +export interface IKeyProvider { + /** + * Get the type of this key provider + */ + getType(): string + + /** + * Get the libp2p PeerId + */ + getPeerId(): PeerId + + /** + * Get the libp2p private key + */ + getLibp2pPrivateKey(): any // libp2p PrivateKey type + + /** + * Get the libp2p public key as Uint8Array + */ + getPublicKey(): Uint8Array + + /** + * Get the Ethereum address derived from the private key + */ + getEthAddress(): string + /** + * Get the Ethereum Wallet derived from the private key + */ + getEthWallet(): Wallet + + /** + * Get the raw private key bytes for EVM signer creation. + * This is used to create ethers Wallet instances. + */ + getRawPrivateKeyBytes(): Uint8Array + + /** + * Encrypts data according to a given algorithm + * @param data data to encrypt + * @param algorithm encryption algorithm AES or ECIES + */ + encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise + + /** + * Decrypts data according to a given algorithm using node keys + * @param data data to decrypt + * @param algorithm decryption algorithm AES or ECIES + */ + decrypt(data: Uint8Array, algorithm: EncryptMethod): Promise + /** + * Encrypts a stream according to a given algorithm using node keys + * @param inputStream - Readable stream to encrypt + * @param algorithm - Encryption algorithm AES or ECIES + * @returns Readable stream with encrypted data + */ + encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable + /** + * Decrypts a stream according to a given algorithm using node keys + * @param inputStream - Readable stream to decrypt + * @param algorithm - Decryption algorithm AES or ECIES + * @returns Readable stream with decrypted data + */ + decryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable + /** + * Decrypts using ethCrypto.decryptWithPrivateKey + * @param encryptedObject + * @returns Decrypted data + */ + ethCryptoDecryptWithPrivateKey(encryptedObject: any): Promise + /** + * Signs message using ethers wallet.signMessage + * @param message - Message to sign + * @returns Signature + */ + signMessage(message: string): Promise + /** + * Cleanup resources if needed (e.g., close connections) + */ + cleanup?(): Promise +} diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index 08db7b98b..4ea7c84ca 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -22,6 +22,16 @@ export interface OceanNodeKeys { publicKey: any privateKey: any ethAddress: string + type?: string | 'raw' | 'gcp-kms' // Key provider type (raw, gcp-kms, etc.) + // Raw private key config (when type is 'raw') + // GCP KMS config (when type is 'gcp-kms') + gcpKmsConfig?: { + projectId: string + location: string + keyRing: string + keyName: string + keyVersion?: string + } } /* eslint-disable no-unused-vars */ export enum dhtFilterMethod { diff --git a/src/@types/commands.ts b/src/@types/commands.ts index e07f77697..4ebb78db4 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -21,6 +21,7 @@ export interface Command { command: string // command name node?: string // if not present it means current node authorization?: string + caller?: string | string[] // added by our node for rate limiting } export interface GetP2PPeerCommand extends Command { diff --git a/src/OceanNode.ts b/src/OceanNode.ts index 0a9f914a5..f32cf7d87 100644 --- a/src/OceanNode.ts +++ b/src/OceanNode.ts @@ -10,6 +10,9 @@ import { GENERIC_EMOJIS, LOG_LEVELS_STR } from './utils/logging/Logger.js' import { BaseHandler } from './components/core/handler/handler.js' import { C2DEngines } from './components/c2d/compute_engines.js' import { Auth } from './components/Auth/index.js' +import { KeyManager } from './components/KeyManager/index.js' +import { BlockchainRegistry } from './components/BlockchainRegistry/index.js' +import { Blockchain } from './utils/blockchain.js' export interface RequestLimiter { requester: string | string[] // IP address or peer ID @@ -41,8 +44,20 @@ export class OceanNode { private db?: Database, private node?: OceanP2P, private provider?: OceanProvider, - private indexer?: OceanIndexer + private indexer?: OceanIndexer, + public keyManager?: KeyManager, + public blockchainRegistry?: BlockchainRegistry ) { + if (keyManager) { + this.keyManager = keyManager + } else { + this.keyManager = new KeyManager(config) + } + if (blockchainRegistry) { + this.blockchainRegistry = blockchainRegistry + } else { + this.blockchainRegistry = new BlockchainRegistry(this.keyManager, config) + } this.coreHandlers = CoreHandlersRegistry.getInstance(this) this.requestMap = new Map() this.config = config @@ -55,7 +70,8 @@ export class OceanNode { if (this.config) { this.escrow = new Escrow( this.config.supportedNetworks, - this.config.claimDurationTimeout + this.config.claimDurationTimeout, + this.blockchainRegistry ) } } @@ -67,11 +83,28 @@ export class OceanNode { node?: OceanP2P, provider?: OceanProvider, indexer?: OceanIndexer, + keyManager?: KeyManager, + blockchainRegistry?: BlockchainRegistry, newInstance: boolean = false ): OceanNode { if (!OceanNode.instance || newInstance) { + if (!keyManager || !blockchainRegistry) { + if (!config) { + throw new Error('KeyManager and BlockchainRegistry are required') + } + keyManager = new KeyManager(config) + blockchainRegistry = new BlockchainRegistry(keyManager, config) + } // prepare compute engines - this.instance = new OceanNode(config, db, node, provider, indexer) + this.instance = new OceanNode( + config, + db, + node, + provider, + indexer, + keyManager, + blockchainRegistry + ) } return this.instance } @@ -94,7 +127,12 @@ export class OceanNode { OCEAN_NODE_LOGGER.error('C2DDatabase is mandatory for compute engines!') return } - this.c2dEngines = new C2DEngines(this.config, this.db.c2d, this.escrow) + this.c2dEngines = new C2DEngines( + this.config, + this.db.c2d, + this.escrow, + this.keyManager + ) await this.c2dEngines.startAllEngines() } } @@ -123,14 +161,6 @@ export class OceanNode { return this.coreHandlers } - public setRemoteCaller(client: string | string[]) { - this.remoteCaller = client - } - - public getRemoteCaller(): string | string[] { - return this.remoteCaller - } - public getRequestMapSize(): number { return this.requestMap.size } @@ -143,12 +173,29 @@ export class OceanNode { return this.auth } + public getKeyManager(): KeyManager { + return this.keyManager + } + + public getBlockchainRegistry(): BlockchainRegistry { + return this.blockchainRegistry + } + + /** + * Get a Blockchain instance for the given chainId. + * Delegates to BlockchainRegistry. + */ + public getBlockchain(chainId: number): Blockchain | null { + return this.blockchainRegistry.getBlockchain(chainId) + } + public setConfig(config: OceanNodeConfig) { this.config = config if (this.config) { this.escrow = new Escrow( this.config.supportedNetworks, - this.config.claimDurationTimeout + this.config.claimDurationTimeout, + this.blockchainRegistry ) } } diff --git a/src/components/BlockchainRegistry/index.ts b/src/components/BlockchainRegistry/index.ts new file mode 100644 index 000000000..c95f2ecba --- /dev/null +++ b/src/components/BlockchainRegistry/index.ts @@ -0,0 +1,89 @@ +import { Blockchain } from '../../utils/blockchain.js' +import { KeyManager } from '../KeyManager/index.js' +import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { RPCS } from '../../@types/blockchain.js' + +/** + * BlockchainRegistry manages Blockchain instances per chainId. + * Provides lazy initialization and centralized access to blockchain connections. + */ +export class BlockchainRegistry { + private blockchains: Map + private keyManager: KeyManager + private config: OceanNodeConfig + + constructor(keyManager: KeyManager, config: OceanNodeConfig) { + this.keyManager = keyManager + this.config = config + this.blockchains = new Map() + } + + /** + * Get or create a Blockchain instance for the given chainId. + * Returns null if the chainId is not supported. + * + * @param chainId - The chain ID to get a Blockchain instance for + * @returns Blockchain instance or null if not supported + */ + getBlockchain(chainId: number): Blockchain | null { + // Check if already initialized + if (this.blockchains.has(chainId)) { + return this.blockchains.get(chainId) + } + + // Check if chainId is supported + const supportedNetworks = this.config.supportedNetworks as RPCS + if (!supportedNetworks || !supportedNetworks[chainId.toString()]) { + return null + } + + // Get network configuration + const networkConfig = supportedNetworks[chainId.toString()] + const { rpc } = networkConfig + const { fallbackRPCs } = networkConfig + + // Create Blockchain instance with new constructor + const blockchain = new Blockchain(this.keyManager, rpc, chainId, fallbackRPCs) + + // Cache the instance + this.blockchains.set(chainId, blockchain) + + return blockchain + } + + /** + * Get all initialized Blockchain instances + * + * @returns Array of all Blockchain instances + */ + getAllBlockchains(): Blockchain[] { + return Array.from(this.blockchains.values()) + } + + /** + * Remove a Blockchain instance from the registry. + * Useful for cleanup or when a network is no longer supported. + * + * @param chainId - The chain ID to remove + */ + removeBlockchain(chainId: number): void { + if (this.blockchains.has(chainId)) { + this.blockchains.delete(chainId) + } + } + + /** + * Clear all Blockchain instances from the registry. + * Useful for cleanup or testing. + */ + clear(): void { + this.blockchains.clear() + } + + /** + * Get the number of initialized Blockchain instances + */ + getBlockchainCount(): number { + return this.blockchains.size + } +} diff --git a/src/components/Indexer/ChainIndexer.ts b/src/components/Indexer/ChainIndexer.ts index c48b487a4..66da722cb 100644 --- a/src/components/Indexer/ChainIndexer.ts +++ b/src/components/Indexer/ChainIndexer.ts @@ -1,5 +1,5 @@ import EventEmitter from 'node:events' -import { JsonRpcApiProvider, Log, Signer } from 'ethers' +import { FallbackProvider, Log, Signer } from 'ethers' import { SupportedNetwork } from '../../@types/blockchain.js' import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' import { isDefined, sleep } from '../../utils/util.js' @@ -163,8 +163,8 @@ export class ChainIndexer { `Initial details for chain ${this.blockchain.getSupportedChain()}: RPCS start block: ${this.rpcDetails.startBlock}, Contract deployment block: ${contractDeploymentBlock}, Crawling start block: ${crawlingStartBlock}` ) - const provider = this.blockchain.getProvider() - const signer = this.blockchain.getSigner() + const provider = await this.blockchain.getProvider() + const signer = await this.blockchain.getSigner() const interval = getCrawlingInterval() let chunkSize = this.rpcDetails.chunkSize || 1 let successfulRetrievalCount = 0 @@ -435,7 +435,7 @@ export class ChainIndexer { * Uses FIFO (First-In, First-Out) order via shift() */ private async processReindexQueue( - provider: JsonRpcApiProvider, + provider: FallbackProvider, signer: Signer ): Promise { while (this.reindexQueue.length > 0) { diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 62b400680..d02a1ae60 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -28,10 +28,10 @@ import { INDEXER_LOGGER } from '../../utils/logging/common.js' import { Blockchain, EVENTS, - getConfiguration, INDEXER_CRAWLING_EVENTS, PROTOCOL_COMMANDS } from '../../utils/index.js' +import { BlockchainRegistry } from '../BlockchainRegistry/index.js' import { CommandStatus, JobStatus } from '../../@types/commands.js' import { buildJobIdentifier, getDeployedContractBlock } from './utils.js' import { create256Hash } from '../../utils/crypt.js' @@ -78,13 +78,19 @@ let numCrawlAttempts = 0 export class OceanIndexer { private db: Database private networks: RPCS + private blockchainRegistry?: BlockchainRegistry private supportedChains: string[] private indexers: Map = new Map() private MIN_REQUIRED_VERSION = '0.2.2' - constructor(db: Database, supportedNetworks: RPCS) { + constructor( + db: Database, + supportedNetworks: RPCS, + blockchainRegistry: BlockchainRegistry + ) { this.db = db this.networks = supportedNetworks + this.blockchainRegistry = blockchainRegistry this.supportedChains = Object.keys(supportedNetworks) INDEXING_QUEUE = [] this.startThreads() @@ -145,12 +151,6 @@ export class OceanIndexer { async startCrawler(blockchain: Blockchain): Promise { if ((await blockchain.isNetworkReady()).ready) { return true - } else { - // try other RPCS if any available (otherwise will just retry the same RPC) - const connectionStatus = await blockchain.tryFallbackRPCs() - if (connectionStatus.ready || (await blockchain.isNetworkReady()).ready) { - return true - } } return false } @@ -218,13 +218,12 @@ export class OceanIndexer { return null } - const config = await getConfiguration() - const blockchain = new Blockchain( - rpcDetails.rpc, - rpcDetails.chainId, - config, - rpcDetails.fallbackRPCs - ) + // Use BlockchainRegistry if available, otherwise fall back to old pattern + const blockchain: Blockchain = this.blockchainRegistry.getBlockchain(chainID) + if (!blockchain) { + INDEXER_LOGGER.error(`Unable to get Blockchain instance for chain: ${chainID}`) + return null + } const canStartIndexer = await this.retryCrawlerWithDelay(blockchain) if (!canStartIndexer) { INDEXER_LOGGER.error(`Cannot start indexer. Check DB and RPC connections!`) diff --git a/src/components/Indexer/processor.ts b/src/components/Indexer/processor.ts index 938f7b3d1..4b8f8be88 100644 --- a/src/components/Indexer/processor.ts +++ b/src/components/Indexer/processor.ts @@ -1,4 +1,4 @@ -import { ethers, Signer, JsonRpcApiProvider, Interface, getAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, Interface, getAddress } from 'ethers' import { BlocksEvents, ProcessingEvents } from '../../@types/blockchain.js' import { EVENTS } from '../../utils/constants.js' import { getConfiguration } from '../../utils/config.js' @@ -58,7 +58,7 @@ function getEventProcessor(eventType: string, chainId: number): BaseEventProcess export const processChunkLogs = async ( logs: readonly ethers.Log[], signer: Signer, - provider: JsonRpcApiProvider, + provider: FallbackProvider, chainId: number ): Promise => { const storeEvents: BlocksEvents = {} @@ -185,7 +185,7 @@ export const processChunkLogs = async ( export const processBlocks = async ( blockLogs: ethers.Log[], signer: Signer, - provider: JsonRpcApiProvider, + provider: FallbackProvider, network: number, lastIndexedBlock: number, count: number diff --git a/src/components/Indexer/processors/BaseProcessor.ts b/src/components/Indexer/processors/BaseProcessor.ts index f21cc9628..6a993d946 100644 --- a/src/components/Indexer/processors/BaseProcessor.ts +++ b/src/components/Indexer/processors/BaseProcessor.ts @@ -4,19 +4,18 @@ import { ZeroAddress, Signer, ethers, - JsonRpcApiProvider, Interface, toUtf8Bytes, hexlify, getBytes, - toUtf8String + toUtf8String, + FallbackProvider } from 'ethers' import { Readable } from 'winston-transport' -import { DecryptDDOCommand } from '../../../@types/commands.js' +import { DecryptDDOCommand, NonceCommand } from '../../../@types/commands.js' import { OceanNode } from '../../../OceanNode.js' import { EVENT_HASHES, PROTOCOL_COMMANDS } from '../../../utils/constants.js' import { timestampToDateTime } from '../../../utils/conversions.js' -import { getConfiguration } from '../../../utils/config.js' import { create256Hash } from '../../../utils/crypt.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -28,7 +27,6 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20TemplateEnterprise.sol/ERC20TemplateEnterprise.json' with { type: 'json' } import { fetchTransactionReceipt } from '../../core/utils/validateOrders.js' import { withRetrial } from '../utils.js' -import { OceanNodeKeys } from '../../../@types/OceanNode.js' export abstract class BaseEventProcessor { protected networkId: number @@ -85,7 +83,7 @@ export abstract class BaseEventProcessor { } protected async getEventData( - provider: JsonRpcApiProvider, + provider: FallbackProvider, transactionHash: string, abi: any, eventType: string @@ -206,14 +204,14 @@ export abstract class BaseEventProcessor { return true } - private async getNonce(decryptorURL: string, keys: OceanNodeKeys) { + private async getNonce(decryptorURL: string, address: string) { try { if (URLUtils.isValidUrl(decryptorURL)) { INDEXER_LOGGER.logMessage( `decryptDDO: Making HTTP request for nonce. DecryptorURL: ${decryptorURL}` ) const nonceResponse = await axios.get( - `${decryptorURL}/api/services/nonce?userAddress=${keys.ethAddress}`, + `${decryptorURL}/api/services/nonce?userAddress=${address}`, { timeout: 20000 } ) return nonceResponse.status === 200 && nonceResponse.data @@ -248,45 +246,32 @@ export abstract class BaseEventProcessor { INDEXER_LOGGER.logMessage( `Decrypting DDO from network: ${this.networkId} created by: ${eventCreator} encrypted by: ${decryptorURL}` ) - const config = await getConfiguration() - const { keys } = config - const nodeId = keys.peerId.toString() - const wallet: ethers.Wallet = new ethers.Wallet(process.env.PRIVATE_KEY as string) - const useTxIdOrContractAddress = txId || contractAddress - - const createSignature = async () => { - const nonce: string = await this.getNonce(decryptorURL, keys) - INDEXER_LOGGER.logMessage( - `decryptDDO: Fetched fresh nonce ${nonce} for decrypt attempt` - ) - const message = String( - useTxIdOrContractAddress + keys.ethAddress + chainId.toString() + nonce - ) - const messageHash = ethers.solidityPackedKeccak256( - ['bytes'], - [ethers.hexlify(ethers.toUtf8Bytes(message))] - ) - const messageHashBytes = ethers.getBytes(messageHash) - const signature = await wallet.signMessage(messageHashBytes) - - const recoveredAddress = ethers.verifyMessage(messageHashBytes, signature) - INDEXER_LOGGER.logMessage( - `decryptDDO: recovered address: ${recoveredAddress}, expected: ${keys.ethAddress}` - ) + const oceanNode = OceanNode.getInstance() + const keyManager = oceanNode.getKeyManager() + const nodeId = keyManager.getPeerId().toString() + const wallet = keyManager.getEthWallet() + const ethAddress = wallet.address - return { nonce, signature } - } + const useTxIdOrContractAddress = txId || contractAddress if (URLUtils.isValidUrl(decryptorURL)) { try { const response = await withRetrial(async () => { - const { nonce, signature } = await createSignature() + const nonce: string = await this.getNonce(decryptorURL, ethAddress) + INDEXER_LOGGER.logMessage( + `decryptDDO: Fetched fresh nonce ${nonce} for decrypt attempt` + ) + + const message = String( + useTxIdOrContractAddress + ethAddress + chainId.toString() + nonce + ) + const signature = await keyManager.signMessage(message) const payload = { transactionId: txId, chainId, - decrypterAddress: keys.ethAddress, + decrypterAddress: ethAddress, dataNftAddress: contractAddress, signature, nonce @@ -365,73 +350,130 @@ export abstract class BaseEventProcessor { throw new Error(message) } } else { - const node = OceanNode.getInstance(config, await getDatabase()) + // const node = OceanNode.getInstance(config, await getDatabase()) if (nodeId === decryptorURL) { - // Fetch nonce and signature for local node path - const { nonce, signature } = await createSignature() + // Fetch nonce and signature from local node + let nonceP2p: string + const getNonceTask: NonceCommand = { + address: ethAddress, + command: PROTOCOL_COMMANDS.NONCE + } + try { + const response = await oceanNode + .getCoreHandlers() + .getHandler(PROTOCOL_COMMANDS.NONCE) + .handle(getNonceTask) + nonceP2p = await streamToString(response.stream as Readable) + } catch (error) { + const message = `Node exception on getting nonce from local nodeId ${nodeId}. Status: ${error.message}` + INDEXER_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, message) + throw new Error(message) + } + INDEXER_LOGGER.debug( + `decryptDDO: Fetched fresh nonce ${nonceP2p} for decrypt attempt from local nodeId ${nodeId}` + ) + + const message = String( + useTxIdOrContractAddress + ethAddress + chainId.toString() + nonceP2p + ) + const signature = await keyManager.signMessage(message) const decryptDDOTask: DecryptDDOCommand = { command: PROTOCOL_COMMANDS.DECRYPT_DDO, transactionId: txId, - decrypterAddress: keys.ethAddress, + decrypterAddress: ethAddress, chainId, encryptedDocument: metadata, documentHash: metadataHash, dataNftAddress: contractAddress, signature, - nonce + nonce: nonceP2p } try { - const response = await node + const response = await oceanNode .getCoreHandlers() .getHandler(PROTOCOL_COMMANDS.DECRYPT_DDO) .handle(decryptDDOTask) ddo = JSON.parse(await streamToString(response.stream as Readable)) } catch (error) { - const message = `Node exception on decrypt DDO. Status: ${error.message}` + const message = `Node exception on decrypt DDO from local nodeId ${nodeId}. Status: ${error.message}` INDEXER_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, message) throw new Error(message) } } else { + // it's a remote node try { - const { nonce, signature } = await createSignature() + const p2pNode = await oceanNode.getP2PNode() + const getNonceTask: NonceCommand = { + address: ethAddress, + command: PROTOCOL_COMMANDS.NONCE + } + let response = await p2pNode.sendTo( + decryptorURL, + JSON.stringify(getNonceTask) + ) - const p2pNode = await node.getP2PNode() + if (response.status.httpStatus !== 200) { + const logMessage = `Node exception on get nonce from remote nodeId ${nodeId}. Status: ${response.status.httpStatus} ${response.status.error}` + INDEXER_LOGGER.warn(logMessage) + throw new Error(logMessage) + } + + if (!response.stream) { + const logMessage = `No stream for get nonce from remote nodeId ${nodeId}. Status: ${response.status.httpStatus} ${response.status.error}` + INDEXER_LOGGER.warn(logMessage) + throw new Error(logMessage) + } + + // Convert stream to Uint8Array + const remoteNonce = await streamToString(response.stream as Readable) + INDEXER_LOGGER.debug( + `decryptDDO: Fetched fresh nonce ${remoteNonce} from remote node ${decryptorURL} for decrypt attempt` + ) + + const messageToSign = String( + useTxIdOrContractAddress + ethAddress + chainId.toString() + remoteNonce + ) + const signature = await keyManager.signMessage(messageToSign) const message = { command: PROTOCOL_COMMANDS.DECRYPT_DDO, transactionId: txId, - decrypterAddress: keys.ethAddress, + decrypterAddress: ethAddress, chainId, encryptedDocument: metadata, documentHash: metadataHash, dataNftAddress: contractAddress, signature, - nonce + nonce: remoteNonce } - const response = await p2pNode.sendTo(decryptorURL, JSON.stringify(message)) + response = await p2pNode.sendTo(decryptorURL, JSON.stringify(message)) if (response.status.httpStatus !== 200) { - throw new Error(`Decrypt failed: ${response.status.error}`) + const logMessage = `Node exception on decryptDDO from remote nodeId ${nodeId}. Status: ${response.status.httpStatus} ${response.status.error}` + INDEXER_LOGGER.warn(logMessage) + throw new Error(logMessage) } if (!response.stream) { - throw new Error('No data received from decrypt') + const logMessage = `No stream for decryptDDO from remote nodeId ${nodeId}. Status: ${response.status.httpStatus} ${response.status.error}` + INDEXER_LOGGER.warn(logMessage) + throw new Error(logMessage) } // Convert stream to Uint8Array const data = await streamToUint8Array(response.stream as Readable) ddo = JSON.parse(uint8ArrayToString(data)) } catch (error) { - const message = `Node exception on decrypt DDO. Status: ${error.message}` + const message = `Exception from remote nodeId ${nodeId}. Status: ${error.message}` INDEXER_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, message) throw new Error(message) } } } } else { - INDEXER_LOGGER.logMessage( + INDEXER_LOGGER.debug( `Decompressing DDO from network: ${this.networkId} created by: ${eventCreator} ecnrypted by: ${decryptorURL}` ) const byteArray = getBytes(metadata) @@ -446,7 +488,7 @@ export abstract class BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider, + provider: FallbackProvider, eventName?: string ): Promise } diff --git a/src/components/Indexer/processors/DispenserActivatedEventProcessor.ts b/src/components/Indexer/processors/DispenserActivatedEventProcessor.ts index 9b05ce3e9..501c0d91a 100644 --- a/src/components/Indexer/processors/DispenserActivatedEventProcessor.ts +++ b/src/components/Indexer/processors/DispenserActivatedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, ZeroAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, ZeroAddress } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -20,7 +20,7 @@ export class DispenserActivatedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { const decodedEventData = await this.getEventData( provider, diff --git a/src/components/Indexer/processors/DispenserCreatedEventProcessor.ts b/src/components/Indexer/processors/DispenserCreatedEventProcessor.ts index d6ae4afd0..84a517bfc 100644 --- a/src/components/Indexer/processors/DispenserCreatedEventProcessor.ts +++ b/src/components/Indexer/processors/DispenserCreatedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager, PriceType } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, ZeroAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, ZeroAddress } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -20,7 +20,7 @@ export class DispenserCreatedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { const decodedEventData = await this.getEventData( provider, diff --git a/src/components/Indexer/processors/DispenserDeactivatedEventProcessor.ts b/src/components/Indexer/processors/DispenserDeactivatedEventProcessor.ts index 6860a813a..c89ef2ed8 100644 --- a/src/components/Indexer/processors/DispenserDeactivatedEventProcessor.ts +++ b/src/components/Indexer/processors/DispenserDeactivatedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, ZeroAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, ZeroAddress } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -20,7 +20,7 @@ export class DispenserDeactivatedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { const decodedEventData = await this.getEventData( provider, diff --git a/src/components/Indexer/processors/ExchangeActivatedEventProcessor.ts b/src/components/Indexer/processors/ExchangeActivatedEventProcessor.ts index f2d00e162..a99ce2f15 100644 --- a/src/components/Indexer/processors/ExchangeActivatedEventProcessor.ts +++ b/src/components/Indexer/processors/ExchangeActivatedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, ZeroAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, ZeroAddress } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -20,7 +20,7 @@ export class ExchangeActivatedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { try { if (!(await isValidFreContract(event.address, chainId, signer))) { diff --git a/src/components/Indexer/processors/ExchangeCreatedEventProcessor.ts b/src/components/Indexer/processors/ExchangeCreatedEventProcessor.ts index e925917bf..d13c0109d 100644 --- a/src/components/Indexer/processors/ExchangeCreatedEventProcessor.ts +++ b/src/components/Indexer/processors/ExchangeCreatedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, ZeroAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, ZeroAddress } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -20,7 +20,7 @@ export class ExchangeCreatedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { try { if (!(await isValidFreContract(event.address, chainId, signer))) { diff --git a/src/components/Indexer/processors/ExchangeDeactivatedEventProcessor.ts b/src/components/Indexer/processors/ExchangeDeactivatedEventProcessor.ts index e4acacf10..d5f02c15c 100644 --- a/src/components/Indexer/processors/ExchangeDeactivatedEventProcessor.ts +++ b/src/components/Indexer/processors/ExchangeDeactivatedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, ZeroAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, ZeroAddress } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -20,7 +20,7 @@ export class ExchangeDeactivatedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { if (!(await isValidFreContract(event.address, chainId, signer))) { INDEXER_LOGGER.error( diff --git a/src/components/Indexer/processors/ExchangeRateChangedEventProcessor.ts b/src/components/Indexer/processors/ExchangeRateChangedEventProcessor.ts index c0171d996..52e2b0ef8 100644 --- a/src/components/Indexer/processors/ExchangeRateChangedEventProcessor.ts +++ b/src/components/Indexer/processors/ExchangeRateChangedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, ZeroAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, ZeroAddress } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -20,7 +20,7 @@ export class ExchangeRateChangedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { try { if (!(await isValidFreContract(event.address, chainId, signer))) { diff --git a/src/components/Indexer/processors/MetadataEventProcessor.ts b/src/components/Indexer/processors/MetadataEventProcessor.ts index 89389b208..84a9e971c 100644 --- a/src/components/Indexer/processors/MetadataEventProcessor.ts +++ b/src/components/Indexer/processors/MetadataEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager, DDO, VersionedDDO } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider, getAddress } from 'ethers' +import { ethers, Signer, FallbackProvider, getAddress } from 'ethers' import { ENVIRONMENT_VARIABLES, EVENTS, @@ -23,7 +23,7 @@ export class MetadataEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider, + provider: FallbackProvider, eventName: string ): Promise { let did = 'did:op' diff --git a/src/components/Indexer/processors/MetadataStateEventProcessor.ts b/src/components/Indexer/processors/MetadataStateEventProcessor.ts index 7b5b2ab29..38a2e022c 100644 --- a/src/components/Indexer/processors/MetadataStateEventProcessor.ts +++ b/src/components/Indexer/processors/MetadataStateEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider } from 'ethers' +import { ethers, Signer, FallbackProvider } from 'ethers' import { EVENTS, MetadataStates } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -13,7 +13,7 @@ export class MetadataStateEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, _signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { INDEXER_LOGGER.logMessage(`Processing metadata state event...`, true) const decodedEventData = await this.getEventData( diff --git a/src/components/Indexer/processors/OrderReusedEventProcessor.ts b/src/components/Indexer/processors/OrderReusedEventProcessor.ts index ab2faf0ea..0ffb5726d 100644 --- a/src/components/Indexer/processors/OrderReusedEventProcessor.ts +++ b/src/components/Indexer/processors/OrderReusedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider } from 'ethers' +import { ethers, Signer, FallbackProvider } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -18,7 +18,7 @@ export class OrderReusedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { const decodedEventData = await this.getEventData( provider, diff --git a/src/components/Indexer/processors/OrderStartedEventProcessor.ts b/src/components/Indexer/processors/OrderStartedEventProcessor.ts index e836078c6..224f0cb7b 100644 --- a/src/components/Indexer/processors/OrderStartedEventProcessor.ts +++ b/src/components/Indexer/processors/OrderStartedEventProcessor.ts @@ -1,5 +1,5 @@ import { DDOManager } from '@oceanprotocol/ddo-js' -import { ethers, Signer, JsonRpcApiProvider } from 'ethers' +import { ethers, Signer, FallbackProvider } from 'ethers' import { EVENTS } from '../../../utils/constants.js' import { getDatabase } from '../../../utils/database.js' import { INDEXER_LOGGER } from '../../../utils/logging/common.js' @@ -13,7 +13,7 @@ export class OrderStartedEventProcessor extends BaseEventProcessor { event: ethers.Log, chainId: number, signer: Signer, - provider: JsonRpcApiProvider + provider: FallbackProvider ): Promise { const decodedEventData = await this.getEventData( provider, diff --git a/src/components/Indexer/utils.ts b/src/components/Indexer/utils.ts index 2be31fb62..84445dfed 100644 --- a/src/components/Indexer/utils.ts +++ b/src/components/Indexer/utils.ts @@ -1,4 +1,4 @@ -import { JsonRpcApiProvider, Signer, ethers, getAddress } from 'ethers' +import { Signer, ethers, getAddress, FallbackProvider } from 'ethers' import ERC721Factory from '@oceanprotocol/contracts/artifacts/contracts/ERC721Factory.sol/ERC721Factory.json' with { type: 'json' } import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC721Template.sol/ERC721Template.json' with { type: 'json' } import { EVENT_HASHES, isDefined } from '../../utils/index.js' @@ -61,7 +61,7 @@ export const getDeployedContractBlock = (network: number) => { return deployedBlock } -export const getNetworkHeight = async (provider: JsonRpcApiProvider) => { +export const getNetworkHeight = async (provider: FallbackProvider) => { const networkHeight = await provider.getBlockNumber() return networkHeight @@ -69,7 +69,7 @@ export const getNetworkHeight = async (provider: JsonRpcApiProvider) => { export const retrieveChunkEvents = async ( signer: Signer, - provider: JsonRpcApiProvider, + provider: FallbackProvider, network: number, lastIndexedBlock: number, count: number diff --git a/src/components/KeyManager/index.ts b/src/components/KeyManager/index.ts new file mode 100644 index 000000000..060d98980 --- /dev/null +++ b/src/components/KeyManager/index.ts @@ -0,0 +1,213 @@ +import type { PeerId } from '@libp2p/interface' +import { Signer, Wallet, FallbackProvider } from 'ethers' +import { IKeyProvider } from '../../@types/KeyManager.js' +import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { RawPrivateKeyProvider } from './providers/RawPrivateKeyProvider.js' +import { EncryptMethod } from '../../@types/fileObject.js' +import { Readable } from 'stream' +/** + * Factory function to create the appropriate key provider based on configuration. + * Provider is initialized in its constructor. + * + * @param config - Key provider configuration + * @returns An initialized key provider instance + */ +export function createKeyProvider(config: OceanNodeConfig): IKeyProvider { + let provider: IKeyProvider + + switch (config.keys.type) { + case 'raw': + provider = new RawPrivateKeyProvider(config) + break + + case 'gcp-kms': + throw new Error('GCP KMS provider not yet implemented') + + default: + throw new Error(`Unsupported key provider type: ${config.keys.type}`) + } + + return provider +} + +/** + * KeyManager centralizes all key management for OceanNode. + * Provides access to peerId, libp2p keys, EVM address, and EVM signers. + * Uses a key provider abstraction to support multiple key sources (raw, GCP KMS, etc.) + */ +export class KeyManager { + private keyProvider: IKeyProvider + private evmSigners: Map // Cache: "chainId-providerKey" -> Signer + + constructor(config: OceanNodeConfig) { + // Determine and create the appropriate key provider based on config.keys.type + + this.keyProvider = createKeyProvider(config) + this.evmSigners = new Map() + } + + /** + * Get the libp2p PeerId + */ + getPeerId(): PeerId { + return this.keyProvider.getPeerId() + } + + /** + * Get the libp2p private key + */ + getLibp2pPrivateKey(): any { + return this.keyProvider.getLibp2pPrivateKey() + } + + /** + * Get the libp2p public key as Uint8Array + */ + getPublicKey(): Uint8Array { + return this.keyProvider.getPublicKey() + } + + /** + * Get the Ethereum Wallet + */ + getEthWallet(): Wallet { + return this.keyProvider.getEthWallet() + } + + /** + * Get the Ethereum address + */ + getEthAddress(): string { + return this.keyProvider.getEthAddress() + } + + /** + * Get the key provider instance + */ + getKeyProvider(): IKeyProvider { + return this.keyProvider + } + + /** + * Get or create an EVM signer for a specific chainId and provider. + * Signers are cached per chainId + * + * @param provider - The JSON-RPC provider to connect the signer to + * @returns An ethers Signer instance + */ + async getEvmSigner(provider: FallbackProvider, chainId?: number): Promise { + // Create a cache key based on chainId and provider URL + // TO DO + if (!chainId) { + const { chainId: networkChainId } = await provider.getNetwork() + chainId = Number(networkChainId) + } + + const cacheKey = `${chainId}` + + // Check if we have a cached signer + if (this.evmSigners.has(cacheKey)) { + let cachedSigner = this.evmSigners.get(cacheKey) + + // If the provider changed, reconnect the signer + if (cachedSigner.provider !== provider) { + cachedSigner = (cachedSigner as Wallet).connect(provider) + this.evmSigners.set(cacheKey, cachedSigner) + } + + return cachedSigner + } + + // Create new signer from private key bytes + const privateKeyBytes = this.keyProvider.getRawPrivateKeyBytes() + const privateKeyHex = Buffer.from(privateKeyBytes).toString('hex') + const signer = new Wallet(privateKeyHex, provider) + + // Cache the signer + this.evmSigners.set(cacheKey, signer) + + return signer + } + + /** + * Clear all cached EVM signers. + * Useful for testing or key rotation scenarios. + */ + clearEvmSignerCache(): void { + this.evmSigners.clear() + } + + /** + * Get the peerId as a string (for compatibility with existing code) + */ + getPeerIdString(): string { + return this.keyProvider.getPeerId().toString() + } + + /** + * This method encrypts data according to a given algorithm using node keys + * @param data data to encrypt + * @param algorithm encryption algorithm AES or ECIES + */ + async encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { + return await this.keyProvider.encrypt(data, algorithm) + } + + /** + * This method decrypts data according to a given algorithm using node keys + * @param data data to decrypt + * @param algorithm decryption algorithm AES or ECIES + */ + async decrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { + return await this.keyProvider.decrypt(data, algorithm) + } + + /** + * Encrypts a stream according to a given algorithm using node keys + * @param inputStream - Readable stream to encrypt + * @param algorithm - Encryption algorithm AES or ECIES + * @returns Readable stream with encrypted data + */ + encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { + return this.keyProvider.encryptStream(inputStream, algorithm) + } + + /** + * Decrypts a stream according to a given algorithm using node keys + * @param inputStream - Readable stream to decrypt + * @param algorithm - Decryption algorithm AES or ECIES + * @returns Readable stream with decrypted data + */ + decryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { + return this.keyProvider.decryptStream(inputStream, algorithm) + } + + /** + * Decrypts using ethCrypto.decryptWithPrivateKey + * @param key + * @param encryptedObject + * @returns Decrypted data + */ + async ethCryptoDecryptWithPrivateKey(encryptedObject: any): Promise { + return await this.keyProvider.ethCryptoDecryptWithPrivateKey(encryptedObject) + } + + /** + * Signs message using ethers wallet.signMessage + * @param message - Message to sign + * @returns Signature + */ + async signMessage(message: string): Promise { + return await this.keyProvider.signMessage(message) + } + + /** + * Cleanup resources (delegates to key provider if it has cleanup method) + */ + async cleanup(): Promise { + if (this.keyProvider.cleanup) { + await this.keyProvider.cleanup() + } + this.clearEvmSignerCache() + } +} diff --git a/src/components/KeyManager/providers/RawPrivateKeyProvider.ts b/src/components/KeyManager/providers/RawPrivateKeyProvider.ts new file mode 100644 index 000000000..f4d3d1a32 --- /dev/null +++ b/src/components/KeyManager/providers/RawPrivateKeyProvider.ts @@ -0,0 +1,252 @@ +import type { PeerId } from '@libp2p/interface' +import { privateKeyFromRaw } from '@libp2p/crypto/keys' +import { peerIdFromPrivateKey } from '@libp2p/peer-id' +import { Wallet, ethers } from 'ethers' +import { IKeyProvider } from '../../../@types/KeyManager.js' +import { hexStringToByteArray } from '../../../utils/index.js' +import { OceanNodeConfig } from '../../../@types/OceanNode.js' +import { EncryptMethod } from '../../../@types/fileObject.js' +import { Readable, Transform } from 'stream' +import * as ethCrypto from 'eth-crypto' +import eciesjs from 'eciesjs' +import crypto from 'crypto' + +/** + * Raw private key provider. + * Loads private key from environment variable or config. + */ +export class RawPrivateKeyProvider implements IKeyProvider { + private peerId: PeerId + private privateKey: any // libp2p PrivateKey type + private publicKey: Uint8Array + private ethWallet: Wallet + private ethAddress: string + + constructor(config: OceanNodeConfig) { + if (!config.keys.privateKey) { + throw new Error('RawPrivateKeyProvider requires keys.privateKey in config') + } + + // Initialize immediately in constructor + const rawPrivateKey = config.keys.privateKey + + // Remove '0x' prefix if present for processing + const privateKeyHex = rawPrivateKey.startsWith('0x') + ? rawPrivateKey.slice(2) + : rawPrivateKey + + // Convert hex string to bytes (hexStringToByteArray expects hex without 0x) + const privateKeyBytes = hexStringToByteArray(privateKeyHex) + + // Create libp2p private key + const key = privateKeyFromRaw(privateKeyBytes) + + // Derive peerId from private key + this.peerId = peerIdFromPrivateKey(key) + this.publicKey = key.publicKey.raw + + // Store keys + this.privateKey = key + + // Derive Ethereum address + // Wallet constructor accepts hex with or without 0x prefix + // We use without 0x to match existing behavior in getPeerIdFromPrivateKey + this.ethWallet = new Wallet(privateKeyHex) + this.ethAddress = this.ethWallet.address + } + + getType(): string { + return 'raw' + } + + getPeerId(): PeerId { + return this.peerId + } + + getLibp2pPrivateKey(): any { + return this.privateKey + } + + getPublicKey(): Uint8Array { + return this.publicKey + } + + getEthWallet(): Wallet { + return this.ethWallet + } + + getEthAddress(): string { + return this.ethAddress + } + + getRawPrivateKeyBytes(): Uint8Array { + return this.privateKey.raw + } + + /** + * This method encrypts data according to a given algorithm using node keys + * @param data data to encrypt + * @param algorithm encryption algorithm AES or ECIES + */ + // eslint-disable-next-line require-await + async encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { + let encryptedData: Buffer + const { privateKey, publicKey } = this + if (algorithm === EncryptMethod.AES) { + // use first 16 bytes of public key as an initialisation vector + const initVector = publicKey.subarray(0, 16) + // creates cipher object, with the given algorithm, key and initialization vector + const cipher = crypto.createCipheriv('aes-256-cbc', privateKey.raw, initVector) + // encoding is ignored because we are working with bytes and want to return a buffer + encryptedData = Buffer.concat([cipher.update(data), cipher.final()]) + } else if (algorithm === EncryptMethod.ECIES) { + const sk = new eciesjs.PrivateKey(privateKey.raw) + // get public key from Elliptic curve + encryptedData = eciesjs.encrypt(sk.publicKey.toHex(), data) + } + return encryptedData + } + + /** + * This method decrypts data according to a given algorithm using node keys + * @param data data to decrypt + * @param algorithm decryption algorithm AES or ECIES + */ + // eslint-disable-next-line require-await + async decrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { + let decryptedData: Buffer + const { privateKey, publicKey } = this + if (algorithm === EncryptMethod.AES) { + // use first 16 bytes of public key as an initialisation vector + const initVector = publicKey.subarray(0, 16) + // creates decipher object, with the given algorithm, key and initialization vector + + const decipher = crypto.createDecipheriv('aes-256-cbc', privateKey.raw, initVector) + + // encoding is ignored because we are working with bytes and want to return a buffer + decryptedData = Buffer.concat([decipher.update(data), decipher.final()]) + } else if (algorithm === EncryptMethod.ECIES) { + const sk = new eciesjs.PrivateKey(privateKey.raw) + decryptedData = eciesjs.decrypt(sk.secret, data) + } + return decryptedData + } + + /** + * Encrypts a stream according to a given algorithm using node keys + * @param inputStream - Readable stream to encrypt + * @param algorithm - Encryption algorithm AES or ECIES + * @returns Readable stream with encrypted data + */ + encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { + const { privateKey, publicKey } = this + + if (algorithm === EncryptMethod.AES) { + // Use first 16 bytes of public key as an initialization vector + const initVector = publicKey.subarray(0, 16) + // Create cipher transform stream + const cipher = crypto.createCipheriv('aes-256-cbc', privateKey.raw, initVector) + + // Pipe input stream through cipher and return the encrypted stream + return inputStream.pipe(cipher) + } else if (algorithm === EncryptMethod.ECIES) { + // ECIES doesn't support streaming, so we need to collect all data first + const chunks: Buffer[] = [] + const collector = new Transform({ + transform(chunk, encoding, callback) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding)) + callback() + }, + flush(callback) { + // Collect all chunks + const data = Buffer.concat(chunks) + // Encrypt using ECIES + const sk = new eciesjs.PrivateKey(privateKey.raw) + const encryptedData = eciesjs.encrypt(sk.publicKey.toHex(), data) + // Push encrypted data as a single chunk + this.push(Buffer.from(encryptedData)) + callback() + } + }) + + return inputStream.pipe(collector) + } else { + throw new Error(`Unsupported encryption algorithm: ${algorithm}`) + } + } + + /** + * Decrypts a stream according to a given algorithm using node keys + * @param inputStream - Readable stream to decrypt + * @param algorithm - Decryption algorithm AES or ECIES + * @returns Readable stream with decrypted data + */ + decryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { + const { privateKey, publicKey } = this + + if (algorithm === EncryptMethod.AES) { + // Use first 16 bytes of public key as an initialization vector + const initVector = publicKey.subarray(0, 16) + // Create decipher transform stream + const decipher = crypto.createDecipheriv('aes-256-cbc', privateKey.raw, initVector) + + // Pipe input stream through decipher and return the decrypted stream + return inputStream.pipe(decipher) + } else if (algorithm === EncryptMethod.ECIES) { + // ECIES doesn't support streaming, so we need to collect all data first + const chunks: Buffer[] = [] + const collector = new Transform({ + transform(chunk, encoding, callback) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding)) + callback() + }, + flush(callback) { + // Collect all chunks + const data = Buffer.concat(chunks) + // Decrypt using ECIES + const sk = new eciesjs.PrivateKey(privateKey.raw) + const decryptedData = eciesjs.decrypt(sk.secret, data) + // Push decrypted data as a single chunk + this.push(Buffer.from(decryptedData)) + callback() + } + }) + + return inputStream.pipe(collector) + } else { + throw new Error(`Unsupported decryption algorithm: ${algorithm}`) + } + } + + /** + * Decrypts using ethCrypto.decryptWithPrivateKey + * @param key + * @param encryptedObject + * @returns Decrypted data + */ + async ethCryptoDecryptWithPrivateKey(encryptedObject: any): Promise { + const { privateKey } = this + const encrypted = ethCrypto.cipher.parse(encryptedObject) + // get the key from configuration + const nodePrivateKey = Buffer.from(privateKey.raw).toString('hex') + const decrypted = await ethCrypto.decryptWithPrivateKey(nodePrivateKey, encrypted) + return decrypted + } + + /** + * Signs message using ethers wallet.signMessage + * @param message - Message to sign + * @returns Signature + */ + async signMessage(message: string): Promise { + const wallet = this.ethWallet + const messageHash = ethers.solidityPackedKeccak256( + ['bytes'], + [ethers.hexlify(ethers.toUtf8Bytes(message))] + ) + const messageHashBytes = ethers.getBytes(messageHash) + const signature = await wallet.signMessage(messageHashBytes) + + return signature + } +} diff --git a/src/components/P2P/handleProtocolCommands.ts b/src/components/P2P/handleProtocolCommands.ts index a4bb5a97b..91b5abefb 100644 --- a/src/components/P2P/handleProtocolCommands.ts +++ b/src/components/P2P/handleProtocolCommands.ts @@ -130,7 +130,7 @@ export async function handleProtocolCommands(stream: Stream, connection: Connect } try { - handler.getOceanNode().setRemoteCaller(remotePeer.toString()) + task.caller = remotePeer.toString() const response: P2PCommandResponse = await handler.handle(task) // Send status first diff --git a/src/components/P2P/index.ts b/src/components/P2P/index.ts index 338ebff1f..cca8e3c74 100644 --- a/src/components/P2P/index.ts +++ b/src/components/P2P/index.ts @@ -39,12 +39,13 @@ import { FindDDOResponse, dhtFilterMethod } from '../../@types/OceanNode.js' +import { KeyManager } from '../KeyManager/index.js' // eslint-disable-next-line camelcase import ipaddr from 'ipaddr.js' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js' import { INDEXER_DDO_EVENT_EMITTER } from '../Indexer/index.js' import { P2P_LOGGER } from '../../utils/logging/common.js' -import { CoreHandlersRegistry } from '../core/handler/coreHandlersRegistry' +import { CoreHandlersRegistry } from '../core/handler/coreHandlersRegistry.js' import { Multiaddr, multiaddr } from '@multiformats/multiaddr' import { LevelDatastore } from 'datastore-level' @@ -76,8 +77,6 @@ export class OceanP2P extends EventEmitter { _connections: {} _protocol: string _publicAddress: string - _publicKey: Uint8Array - _privateKey: Uint8Array _analyzeRemoteResponse: Transform _pendingAdvertise: string[] = [] private _ddoDHT: DDOCache @@ -88,10 +87,12 @@ export class OceanP2P extends EventEmitter { private _idx: number private readonly db: Database private readonly _config: OceanNodeConfig + private readonly keyManager: KeyManager private coreHandlers: CoreHandlersRegistry - constructor(config: OceanNodeConfig, db?: Database) { + constructor(config: OceanNodeConfig, keyManager: KeyManager, db?: Database) { super() this._config = config + this.keyManager = keyManager this.db = db this._ddoDHT = { updated: new Date().getTime(), @@ -284,11 +285,9 @@ export class OceanP2P extends EventEmitter { async createNode(config: OceanNodeConfig): Promise { try { - this._publicAddress = config.keys.peerId.toString() + this._publicAddress = this.keyManager.getPeerIdString() P2P_LOGGER.info(`Starting P2P Node with peerID: ${this._publicAddress}`) - this._publicKey = config.keys.publicKey - this._privateKey = config.keys.privateKey.raw /** @type {import('libp2p').Libp2pOptions} */ // start with some default, overwrite based on config later const bindInterfaces = [] @@ -408,7 +407,7 @@ export class OceanP2P extends EventEmitter { let options = { addresses, datastore: store, - privateKey: config.keys.privateKey, + privateKey: this.keyManager.getLibp2pPrivateKey(), transports, streamMuxers: [yamux()], connectionEncrypters: [ @@ -952,7 +951,7 @@ export class OceanP2P extends EventEmitter { } getPeerId(): string { - return this._config.keys.peerId.toString() + return this.keyManager.getPeerIdString() } getDDOCache(): DDOCache { diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index bbd04b5b6..13e49a6a4 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -20,15 +20,27 @@ import type { import { C2DClusterType } from '../../@types/C2D/C2D.js' import { C2DDatabase } from '../database/C2DDatabase.js' import { Escrow } from '../core/utils/escrow.js' +import { KeyManager } from '../KeyManager/index.js' export abstract class C2DEngine { private clusterConfig: C2DClusterInfo public db: C2DDatabase public escrow: Escrow - public constructor(cluster: C2DClusterInfo, db: C2DDatabase, escrow: Escrow) { + public keyManager: KeyManager + public constructor( + cluster: C2DClusterInfo, + db: C2DDatabase, + escrow: Escrow, + keyManager: KeyManager + ) { this.clusterConfig = cluster this.db = db this.escrow = escrow + this.keyManager = keyManager + } + + getKeyManager(): KeyManager { + return this.keyManager } getC2DConfig(): C2DClusterInfo { @@ -454,21 +466,14 @@ export abstract class C2DEngine { chainId: number, token: string ): ComputeResourcesPricingInfo[] { - console.log('getEnvPricesForToken') - console.log(env) if (!env.fees || !(chainId in env.fees) || !env.fees[chainId]) { return null } - console.log(env.fees) for (const fee of env.fees[chainId]) { - console.log(fee) - console.log(fee.feeToken) - console.log(token) // eslint-disable-next-line security/detect-possible-timing-attacks if (fee.feeToken === token) { - console.log('Found') return fee.prices - } else console.log('NOT Found') + } } return null diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 289aa411c..50a71941d 100644 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -46,6 +46,7 @@ import { CORE_LOGGER } from '../../utils/logging/common.js' import { AssetUtils } from '../../utils/asset.js' import { FindDdoHandler } from '../core/handler/ddoHandler.js' import { OceanNode } from '../../OceanNode.js' +import { KeyManager } from '../KeyManager/index.js' import { decryptFilesObject, omitDBComputeFieldsFromComputeJob } from './index.js' import { ValidateParams } from '../httpRoutes/validateCommands.js' import { Service } from '@oceanprotocol/ddo-js' @@ -60,8 +61,13 @@ export class C2DEngineDocker extends C2DEngine { private jobImageSizes: Map = new Map() private static DEFAULT_DOCKER_REGISTRY = 'https://registry-1.docker.io' - public constructor(clusterConfig: C2DClusterInfo, db: C2DDatabase, escrow: Escrow) { - super(clusterConfig, db, escrow) + public constructor( + clusterConfig: C2DClusterInfo, + db: C2DDatabase, + escrow: Escrow, + keyManager: KeyManager + ) { + super(clusterConfig, db, escrow, keyManager) this.docker = null if (clusterConfig.connection.socketPath) { @@ -110,7 +116,6 @@ export class C2DEngineDocker extends C2DEngine { // since we cannot connect to docker, we cannot start the engine -> no envs return } - // console.log(sysinfo) let fees: ComputeEnvFeesStructure = null const supportedChains: number[] = [] if (config.supportedNetworks) { @@ -120,7 +125,6 @@ export class C2DEngineDocker extends C2DEngine { } for (const feeChain of Object.keys(envConfig.fees)) { // for (const feeConfig of envConfig.fees) { - // console.log(feeChain) if (supportedChains.includes(parseInt(feeChain))) { if (fees === null) fees = {} if (!(feeChain in fees)) fees[feeChain] = [] @@ -163,7 +167,7 @@ export class C2DEngineDocker extends C2DEngine { this.envs.push({ id: '', // this.getC2DConfig().hash + '-' + create256Hash(JSON.stringify(this.envs[i])), runningJobs: 0, - consumerAddress: config.keys.ethAddress, + consumerAddress: this.getKeyManager().getEthAddress(), platform: { architecture: sysinfo.Architecture, os: sysinfo.OSType @@ -548,7 +552,6 @@ export class C2DEngineDocker extends C2DEngine { } else { // already built, we need to validate it const validation = await C2DEngineDocker.checkDockerImage(image, env.platform) - console.log('Validation: ', validation) if (!validation.valid) throw new Error( `Cannot find image ${image} for ${env.platform.architecture}. Maybe it does not exist or it's build for other arhitectures.` @@ -908,8 +911,10 @@ export class C2DEngineDocker extends C2DEngine { // eslint-disable-next-line require-await private async processJob(job: DBComputeJob) { - console.log(`Process job started: [STATUS: ${job.status}: ${job.statusText}]`) - console.log(job) + CORE_LOGGER.info( + `Process job ${job.jobId} started: [STATUS: ${job.status}: ${job.statusText}]` + ) + // has to : // - monitor running containers and stop them if over limits // - monitor disc space and clean up @@ -1072,7 +1077,6 @@ export class C2DEngineDocker extends C2DEngine { } const container = await this.createDockerContainer(containerInfo, true) if (container) { - console.log('Container created: ', container) job.status = C2DStatusNumber.Provisioning job.statusText = C2DStatusText.Provisioning await this.db.updateJob(job) @@ -1090,8 +1094,6 @@ export class C2DEngineDocker extends C2DEngine { if (job.status === C2DStatusNumber.Provisioning) { // download algo & assets const ret = await this.uploadData(job) - console.log('Upload data') - console.log(ret) job.status = ret.status job.statusText = ret.statusText if (job.status !== C2DStatusNumber.RunningAlgorithm) { @@ -1109,10 +1111,7 @@ export class C2DEngineDocker extends C2DEngine { let details try { container = await this.docker.getContainer(job.jobId + '-algoritm') - console.log(`Container retrieved: ${JSON.stringify(container)}`) details = await container.inspect() - console.log('Container inspect') - console.log(details) } catch (e) { console.error( 'Could not retrieve container: ' + @@ -1151,11 +1150,9 @@ export class C2DEngineDocker extends C2DEngine { '/data/logs/algorithm.log' writeFileSync(algoLogFile, String(e.message)) } catch (e) { - console.log('Failed to write') - console.log(e) + CORE_LOGGER.error('Failed to write algorithm log file: ' + e.message) } - console.error('could not start container: ' + e.message) - console.log(e) + CORE_LOGGER.error('Could not start container: ' + e.message) job.status = C2DStatusNumber.AlgorithmFailed job.statusText = C2DStatusText.AlgorithmFailed @@ -1173,24 +1170,22 @@ export class C2DEngineDocker extends C2DEngine { return } - console.log('running, need to stop it?') const timeNow = Date.now() / 1000 const expiry = parseFloat(job.algoStartTimestamp) + job.maxJobDuration - console.log('timeNow: ' + timeNow + ' , Expiry: ' + expiry) + CORE_LOGGER.debug( + 'container running since timeNow: ' + timeNow + ' , Expiry: ' + expiry + ) if (timeNow > expiry || job.stopRequested) { // we need to stop the container // make sure is running - console.log('We need to stop') - console.log(details.State.Running) if (details.State.Running === true) { try { await container.stop() } catch (e) { // we should never reach this, unless the container is already stopped or deleted by someone else - console.log(e) + CORE_LOGGER.debug('Could not stop container: ' + e.message) } } - console.log('Stopped') job.isStarted = false job.status = C2DStatusNumber.PublishingResults job.statusText = C2DStatusText.PublishingResults @@ -1218,9 +1213,8 @@ export class C2DEngineDocker extends C2DEngine { let container try { container = await this.docker.getContainer(job.jobId + '-algoritm') - console.log(`Container retrieved: ${JSON.stringify(container)}`) } catch (e) { - console.error('Could not retrieve container: ' + e.message) + CORE_LOGGER.debug('Could not retrieve container: ' + e.message) job.isRunning = false job.dateFinished = String(Date.now() / 1000) try { @@ -1228,8 +1222,7 @@ export class C2DEngineDocker extends C2DEngine { this.getC2DConfig().tempFolder + '/' + job.jobId + '/data/logs/algorithm.log' writeFileSync(algoLogFile, String(e.message)) } catch (e) { - console.log('Failed to write') - console.log(e) + CORE_LOGGER.error('Failed to write algorithm log file: ' + e.message) } await this.db.updateJob(job) await this.cleanupJob(job) @@ -1254,7 +1247,7 @@ export class C2DEngineDocker extends C2DEngine { ) } } catch (e) { - console.log(e) + CORE_LOGGER.error('Failed to get outputs archive: ' + e.message) job.status = C2DStatusNumber.ResultsUploadFailed job.statusText = C2DStatusText.ResultsUploadFailed } @@ -1314,7 +1307,7 @@ export class C2DEngineDocker extends C2DEngine { proof ) } catch (e) { - console.log(e) + CORE_LOGGER.error('Failed to claim lock: ' + e.message) } } else { // release the lock, we are not getting paid @@ -1326,7 +1319,7 @@ export class C2DEngineDocker extends C2DEngine { job.owner ) } catch (e) { - console.log(e) + CORE_LOGGER.error('Failed to release lock: ' + e.message) } } if (txId) { @@ -1359,7 +1352,7 @@ export class C2DEngineDocker extends C2DEngine { try { await volume.remove() } catch (e) { - console.log(e) + CORE_LOGGER.error('Failed to remove volume: ' + e.message) } } } catch (e) { @@ -1371,7 +1364,7 @@ export class C2DEngineDocker extends C2DEngine { try { await this.docker.getImage(image).remove({ force: true }) } catch (e) { - console.log('Could not delete image: ' + image + ' : ' + e.message) + CORE_LOGGER.error('Could not delete image: ' + image + ' : ' + e.message) } } } @@ -1567,7 +1560,6 @@ export class C2DEngineDocker extends C2DEngine { if (progress.id) logText += progress.id + ' : ' + progress.status else logText = progress.status CORE_LOGGER.debug("Pulling image for jobId '" + job.jobId + "': " + logText) - console.log(progress) appendFileSync(imageLogFile, logText + '\n') } ) @@ -1825,7 +1817,6 @@ export class C2DEngineDocker extends C2DEngine { const asset = job.assets[i] let storage = null let fileInfo = null - console.log('checking now asset: ', i) appendFileSync(configLogPath, `Downloading asset ${i} to /data/inputs/\n`) // without this check it would break if no fileObject is present if (asset.fileObject) { @@ -1951,13 +1942,9 @@ export class C2DEngineDocker extends C2DEngine { try { // await container2.putArchive(destination, { - const stream = await container.putArchive(destination, { + await container.putArchive(destination, { path: '/data' }) - console.log('PutArchive') - console.log(stream) - - console.log('Done uploading') } catch (e) { appendFileSync( configLogPath, @@ -2000,7 +1987,6 @@ export class C2DEngineDocker extends C2DEngine { private async makeJobFolders(job: DBComputeJob) { try { const baseFolder = this.getC2DConfig().tempFolder + '/' + job.jobId - console.log('BASE FOLDER: ' + baseFolder) if (!existsSync(baseFolder)) mkdirSync(baseFolder) if (!existsSync(baseFolder + '/data')) mkdirSync(baseFolder + '/data') if (!existsSync(baseFolder + '/data/inputs')) mkdirSync(baseFolder + '/data/inputs') diff --git a/src/components/c2d/compute_engines.ts b/src/components/c2d/compute_engines.ts index b0bd6ad91..24d7cd46b 100644 --- a/src/components/c2d/compute_engines.ts +++ b/src/components/c2d/compute_engines.ts @@ -4,10 +4,16 @@ import { C2DEngineDocker } from './compute_engine_docker.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { C2DDatabase } from '../database/C2DDatabase.js' import { Escrow } from '../core/utils/escrow.js' +import { KeyManager } from '../KeyManager/index.js' export class C2DEngines { public engines: C2DEngine[] - public constructor(config: OceanNodeConfig, db: C2DDatabase, escrow: Escrow) { + public constructor( + config: OceanNodeConfig, + db: C2DDatabase, + escrow: Escrow, + keyManager: KeyManager + ) { // let's see what engines do we have and initialize them one by one // for docker, we need to add the "free" @@ -17,7 +23,7 @@ export class C2DEngines { this.engines = [] for (const cluster of config.c2dClusters) { if (cluster.type === C2DClusterType.DOCKER) { - this.engines.push(new C2DEngineDocker(cluster, db, escrow)) + this.engines.push(new C2DEngineDocker(cluster, db, escrow, keyManager)) } } } diff --git a/src/components/c2d/index.ts b/src/components/c2d/index.ts index 1d0a9c065..87f4c1300 100644 --- a/src/components/c2d/index.ts +++ b/src/components/c2d/index.ts @@ -1,26 +1,29 @@ import { deleteKeysFromObject, sanitizeServiceFiles } from '../../utils/util.js' -import { decrypt } from '../../utils/crypt.js' import { BaseFileObject, EncryptMethod } from '../../@types/fileObject.js' import { CORE_LOGGER } from '../../utils/logging/common.js' import { ComputeJob, DBComputeJob } from '../../@types/index.js' +import { OceanNode } from '../../OceanNode.js' export { C2DEngine } from './compute_engine_base.js' export async function decryptFilesObject( serviceFiles: any ): Promise { + const node = OceanNode.getInstance() + try { // 2. Decrypt the url - const decryptedUrlBytes = await decrypt( - Uint8Array.from(Buffer.from(sanitizeServiceFiles(serviceFiles), 'hex')), - EncryptMethod.ECIES - ) + const decryptedUrlBytes = await node + .getKeyManager() + .decrypt( + Uint8Array.from(Buffer.from(sanitizeServiceFiles(serviceFiles), 'hex')), + EncryptMethod.ECIES + ) // 3. Convert the decrypted bytes back to a string const decryptedFilesString = Buffer.from(decryptedUrlBytes).toString() const decryptedFileArray = JSON.parse(decryptedFilesString) - console.log('decryptedFileArray: ', decryptedFileArray) return decryptedFileArray.files[0] } catch (err) { CORE_LOGGER.error('Error decrypting files object: ' + err.message) diff --git a/src/components/core/admin/adminHandler.ts b/src/components/core/admin/adminHandler.ts index 394b7c7bd..406cd8c31 100644 --- a/src/components/core/admin/adminHandler.ts +++ b/src/components/core/admin/adminHandler.ts @@ -17,7 +17,7 @@ export abstract class AdminCommandHandler implements IValidateAdminCommandHandler { async verifyParamsAndRateLimits(task: AdminCommand): Promise { - if (!(await this.checkRateLimit())) { + if (!(await this.checkRateLimit(task.caller))) { return buildRateLimitReachedResponse() } // then validate the command arguments diff --git a/src/components/core/admin/collectFeesHandler.ts b/src/components/core/admin/collectFeesHandler.ts index 17bd15124..e101eab66 100644 --- a/src/components/core/admin/collectFeesHandler.ts +++ b/src/components/core/admin/collectFeesHandler.ts @@ -11,12 +11,8 @@ import { buildInvalidRequestMessage, validateCommandParameters } from '../../httpRoutes/validateCommands.js' -import { - getConfiguration, - checkSupportedChainId, - Blockchain -} from '../../../utils/index.js' -import { parseUnits, Contract, ZeroAddress, isAddress, Wallet } from 'ethers' +import { getConfiguration, checkSupportedChainId } from '../../../utils/index.js' +import { parseUnits, Contract, ZeroAddress, isAddress } from 'ethers' import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20Template.sol/ERC20Template.json' with { type: 'json' } import { CORE_LOGGER } from '../../../utils/logging/common.js' import { Readable } from 'stream' @@ -49,7 +45,8 @@ export class CollectFeesHandler extends AdminCommandHandler { return buildInvalidParametersResponse(validation) } const config = await getConfiguration() - if (task.node && task.node !== config.keys.peerId.toString()) { + const keyManager = this.nodeInstance.getKeyManager() + if (task.node && task.node !== keyManager.getPeerIdString()) { const msg: string = `Cannot run this command ${JSON.stringify( task )} on a different node.` @@ -64,10 +61,16 @@ export class CollectFeesHandler extends AdminCommandHandler { } try { - const { rpc, chainId, fallbackRPCs } = config.supportedNetworks[task.chainId] - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - const provider = blockchain.getProvider() - const providerWallet = blockchain.getSigner() as Wallet + const { chainId } = config.supportedNetworks[task.chainId] + const oceanNode = this.getOceanNode() + const blockchain = oceanNode.getBlockchain(chainId) + if (!blockchain) { + return buildErrorResponse( + `Blockchain instance not available for chain ${chainId}` + ) + } + const provider = await blockchain.getProvider() + const providerWallet = blockchain.getWallet() const providerWalletAddress = await providerWallet.getAddress() const ammountInEther = task.tokenAmount ? parseUnits(task.tokenAmount.toString(), 'ether') @@ -90,7 +93,7 @@ export class CollectFeesHandler extends AdminCommandHandler { } receipt = await blockchain.sendTransaction( - providerWallet, + await blockchain.getSigner(), task.destinationAddress.toLowerCase(), ammountInEther ) @@ -98,7 +101,7 @@ export class CollectFeesHandler extends AdminCommandHandler { const token = new Contract( task.tokenAddress.toLowerCase(), ERC20Template.abi, - providerWallet + await blockchain.getSigner() ) const tokenAmount = task.tokenAmount ? parseUnits(task.tokenAmount.toString(), 'ether') diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index ac77953ba..607dd59a8 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -13,11 +13,10 @@ import { isERC20Template4Active } from '../../../utils/asset.js' import { verifyProviderFees, createProviderFee } from '../utils/feesHandler.js' -import { Blockchain } from '../../../utils/blockchain.js' import { validateOrderTransaction } from '../utils/validateOrders.js' import { EncryptMethod } from '../../../@types/fileObject.js' -import { decrypt } from '../../../utils/crypt.js' + import { ValidateParams, buildInvalidRequestMessage, @@ -270,8 +269,18 @@ export class ComputeInitializeHandler extends CommandHandler { } } const config = await getConfiguration() - const { rpc, chainId, fallbackRPCs } = config.supportedNetworks[ddoChainId] - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) + const { chainId } = config.supportedNetworks[ddoChainId] + const oceanNode = this.getOceanNode() + const blockchain = oceanNode.getBlockchain(chainId) + if (!blockchain) { + return { + stream: null, + status: { + httpStatus: 400, + error: `Initialize Compute: Blockchain instance not available for chain ${chainId}` + } + } + } const { ready, error } = await blockchain.isNetworkReady() if (!ready) { return { @@ -300,7 +309,7 @@ export class ComputeInitializeHandler extends CommandHandler { accessGrantedDDOLevel = await checkCredentials( task.consumerAddress, credentials as Credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } if (!accessGrantedDDOLevel) { @@ -347,7 +356,7 @@ export class ComputeInitializeHandler extends CommandHandler { accessGrantedServiceLevel = await checkCredentials( task.consumerAddress, service.credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } @@ -394,7 +403,7 @@ export class ComputeInitializeHandler extends CommandHandler { } } - const signer = blockchain.getSigner() + const signer = await blockchain.getSigner() // check if oasis evm or similar const confidentialEVM = isConfidentialChainDDO(BigInt(ddo.chainId), service) @@ -402,10 +411,14 @@ export class ComputeInitializeHandler extends CommandHandler { let canDecrypt = false try { if (!confidentialEVM) { - await decrypt( - Uint8Array.from(Buffer.from(sanitizeServiceFiles(service.files), 'hex')), - EncryptMethod.ECIES - ) + await node + .getKeyManager() + .decrypt( + Uint8Array.from( + Buffer.from(sanitizeServiceFiles(service.files), 'hex') + ), + EncryptMethod.ECIES + ) canDecrypt = true } else { // TODO 'Initialize compute on confidential EVM! @@ -459,7 +472,7 @@ export class ComputeInitializeHandler extends CommandHandler { } } - const provider = blockchain.getProvider() + const provider = await blockchain.getProvider() result.datatoken = service.datatokenAddress result.chainId = ddoChainId // start with assumption than we need new providerfees @@ -478,7 +491,7 @@ export class ComputeInitializeHandler extends CommandHandler { service.datatokenAddress, AssetUtils.getServiceIndexById(ddo, service.id), service.timeout, - blockchain.getSigner() + await blockchain.getSigner() ) if (paymentValidation.isValid === true) { // order is valid, so let's check providerFees diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 5569a9896..91fb2eaad 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -6,6 +6,7 @@ import { PaidComputeStartCommand } from '../../../@types/commands.js' import { CommandHandler } from '../handler/handler.js' +import { OceanNode } from '../../../OceanNode.js' import { generateUniqueID, getAlgoChecksums, validateAlgoForDataset } from './utils.js' import { ValidateParams, @@ -25,9 +26,7 @@ import { ComputeAccessList, ComputeResourceRequestWithPrice } from '../../../@types/C2D/C2D.js' -import { decrypt } from '../../../utils/crypt.js' // import { verifyProviderFees } from '../utils/feesHandler.js' -import { Blockchain } from '../../../utils/blockchain.js' import { validateOrderTransaction } from '../utils/validateOrders.js' import { getConfiguration, isPolicyServerConfigured } from '../../../utils/index.js' import { sanitizeServiceFiles } from '../../../utils/util.js' @@ -155,7 +154,11 @@ export class PaidComputeStartHandler extends CommandHandler { const { algorithm } = task const config = await getConfiguration() - const accessGranted = await validateAccess(task.consumerAddress, env.access) + const accessGranted = await validateAccess( + task.consumerAddress, + env.access, + this.getOceanNode() + ) if (!accessGranted) { return { stream: null, @@ -191,7 +194,6 @@ export class PaidComputeStartHandler extends CommandHandler { const policyServer = new PolicyServer() // check algo for (const elem of [...[task.algorithm], ...task.datasets]) { - console.log(elem) const result: any = { validOrder: false } if ('documentId' in elem && elem.documentId) { result.did = elem.documentId @@ -226,8 +228,18 @@ export class PaidComputeStartHandler extends CommandHandler { } } const config = await getConfiguration() - const { rpc, chainId, fallbackRPCs } = config.supportedNetworks[ddoChainId] - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) + const { chainId } = config.supportedNetworks[ddoChainId] + const oceanNode = this.getOceanNode() + const blockchain = oceanNode.getBlockchain(chainId) + if (!blockchain) { + return { + stream: null, + status: { + httpStatus: 400, + error: `Start Compute: Blockchain instance not available for chain ${chainId}` + } + } + } const { ready, error } = await blockchain.isNetworkReady() if (!ready) { return { @@ -239,7 +251,7 @@ export class PaidComputeStartHandler extends CommandHandler { } } - const signer = blockchain.getSigner() + const signer = await blockchain.getSigner() // check credentials (DDO level) let accessGrantedDDOLevel: boolean if (credentials) { @@ -258,7 +270,7 @@ export class PaidComputeStartHandler extends CommandHandler { accessGrantedDDOLevel = await checkCredentials( task.consumerAddress, credentials as Credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } if (!accessGrantedDDOLevel) { @@ -305,7 +317,7 @@ export class PaidComputeStartHandler extends CommandHandler { accessGrantedServiceLevel = await checkCredentials( task.consumerAddress, service.credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } @@ -330,10 +342,14 @@ export class PaidComputeStartHandler extends CommandHandler { let canDecrypt = false try { if (!confidentialEVM) { - await decrypt( - Uint8Array.from(Buffer.from(sanitizeServiceFiles(service.files), 'hex')), - EncryptMethod.ECIES - ) + await node + .getKeyManager() + .decrypt( + Uint8Array.from( + Buffer.from(sanitizeServiceFiles(service.files), 'hex') + ), + EncryptMethod.ECIES + ) canDecrypt = true } else { // TODO 'Start compute on confidential EVM!' @@ -401,7 +417,7 @@ export class PaidComputeStartHandler extends CommandHandler { } } - const provider = blockchain.getProvider() + const provider = await blockchain.getProvider() result.datatoken = service.datatokenAddress result.chainId = ddoChainId @@ -425,7 +441,7 @@ export class PaidComputeStartHandler extends CommandHandler { service.datatokenAddress, AssetUtils.getServiceIndexById(ddo, service.id), service.timeout, - blockchain.getSigner() + await blockchain.getSigner() ) if (paymentValidation.isValid === false) { const error = `TxId Service ${elem.transferTxId} is not valid for DDO ${elem.documentId} and service ${service.id}` @@ -695,8 +711,18 @@ export class FreeComputeStartHandler extends CommandHandler { } } const config = await getConfiguration() - const { rpc, chainId, fallbackRPCs } = config.supportedNetworks[ddoChainId] - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) + const { chainId } = config.supportedNetworks[ddoChainId] + const oceanNode = this.getOceanNode() + const blockchain = oceanNode.getBlockchain(chainId) + if (!blockchain) { + return { + stream: null, + status: { + httpStatus: 400, + error: `Start Compute: Blockchain instance not available for chain ${chainId}` + } + } + } const { ready, error } = await blockchain.isNetworkReady() if (!ready) { return { @@ -726,7 +752,7 @@ export class FreeComputeStartHandler extends CommandHandler { accessGrantedDDOLevel = await checkCredentials( task.consumerAddress, credentials as Credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } if (!accessGrantedDDOLevel) { @@ -773,7 +799,7 @@ export class FreeComputeStartHandler extends CommandHandler { accessGrantedServiceLevel = await checkCredentials( task.consumerAddress, service.credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } @@ -803,7 +829,11 @@ export class FreeComputeStartHandler extends CommandHandler { } } try { - const accessGranted = await validateAccess(task.consumerAddress, env.free.access) + const accessGranted = await validateAccess( + task.consumerAddress, + env.free.access, + this.getOceanNode() + ) if (!accessGranted) { return { stream: null, @@ -850,7 +880,6 @@ export class FreeComputeStartHandler extends CommandHandler { } } } - // console.log(task.resources) /* return { stream: null, @@ -911,7 +940,8 @@ export class FreeComputeStartHandler extends CommandHandler { async function validateAccess( consumerAddress: string, - access: ComputeAccessList | undefined + access: ComputeAccessList | undefined, + oceanNode: OceanNode ): Promise { if (!access) { return true @@ -931,10 +961,17 @@ async function validateAccess( const config = await getConfiguration() const { supportedNetworks } = config for (const chain of Object.keys(access.accessLists)) { - const { rpc, chainId, fallbackRPCs } = supportedNetworks[chain] + const { chainId } = supportedNetworks[chain] try { - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - const signer = blockchain.getSigner() + const blockchain = oceanNode.getBlockchain(chainId) + if (!blockchain) { + CORE_LOGGER.logMessage( + `Blockchain instance not available for chain ${chainId}, skipping access list check`, + true + ) + continue + } + const signer = await blockchain.getSigner() for (const accessListAddress of access.accessLists[chain]) { const hasAccess = await checkAddressOnAccessList( accessListAddress, diff --git a/src/components/core/handler/ddoHandler.ts b/src/components/core/handler/ddoHandler.ts index a6e070628..3438fcfae 100644 --- a/src/components/core/handler/ddoHandler.ts +++ b/src/components/core/handler/ddoHandler.ts @@ -1,8 +1,9 @@ import { CommandHandler } from './handler.js' +import { OceanNode } from '../../../OceanNode.js' import { EVENTS, MetadataStates, PROTOCOL_COMMANDS } from '../../../utils/constants.js' import { P2PCommandResponse, FindDDOResponse } from '../../../@types/index.js' import { Readable } from 'stream' -import { decrypt, create256Hash } from '../../../utils/crypt.js' +import { create256Hash } from '../../../utils/crypt.js' import { hasCachedDDO, sortFindDDOResults, @@ -13,7 +14,6 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../../utils/logging/Logger.js' import { sleep, readStream, streamToUint8Array } from '../../../utils/util.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' -import { Blockchain } from '../../../utils/blockchain.js' import { ethers, isAddress } from 'ethers' import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC721Template.sol/ERC721Template.json' with { type: 'json' } // import lzma from 'lzma-native' @@ -131,14 +131,14 @@ export class DecryptDdoHandler extends CommandHandler { } } } - + const ourEthAddress = this.getOceanNode().getKeyManager().getEthAddress() if (config.authorizedDecrypters.length > 0) { // allow if on authorized list or it is own node if ( !config.authorizedDecrypters .map((address) => address?.toLowerCase()) .includes(decrypterAddress?.toLowerCase()) && - decrypterAddress?.toLowerCase() !== config.keys.ethAddress?.toLowerCase() + decrypterAddress?.toLowerCase() !== ourEthAddress.toLowerCase() ) { CORE_LOGGER.logMessage('Decrypt DDO: Decrypter not authorized', true) return { @@ -151,12 +151,17 @@ export class DecryptDdoHandler extends CommandHandler { } } - const blockchain = new Blockchain( - supportedNetwork.rpc, - supportedNetwork.chainId, - config, - supportedNetwork.fallbackRPCs - ) + const oceanNode = this.getOceanNode() + const blockchain = oceanNode.getBlockchain(supportedNetwork.chainId) + if (!blockchain) { + return { + stream: null, + status: { + httpStatus: 400, + error: `Decrypt DDO: Blockchain instance not available for chain ${supportedNetwork.chainId}` + } + } + } const { ready, error } = await blockchain.isNetworkReady() if (!ready) { return { @@ -168,8 +173,8 @@ export class DecryptDdoHandler extends CommandHandler { } } - const provider = blockchain.getProvider() - const signer = blockchain.getSigner() + const provider = await blockchain.getProvider() + const signer = await blockchain.getSigner() // note: "getOceanArtifactsAdresses()"" is broken for at least optimism sepolia // if we do: artifactsAddresses[supportedNetwork.network] // because on the contracts we have "optimism_sepolia" instead of "optimism-sepolia" @@ -313,7 +318,9 @@ export class DecryptDdoHandler extends CommandHandler { // check if DDO is ECIES encrypted if ((flags & 2) !== 0) { try { - decryptedDocument = await decrypt(encryptedDocument, EncryptMethod.ECIES) + decryptedDocument = await oceanNode + .getKeyManager() + .decrypt(encryptedDocument, EncryptMethod.ECIES) } catch (error) { CORE_LOGGER.logMessage(`Decrypt DDO: error ${error}`, true) return { @@ -537,7 +544,7 @@ export class FindDdoHandler extends CommandHandler { const processDDOResponse = async (peer: string, data: Uint8Array) => { try { const ddo: any = JSON.parse(uint8ArrayToString(data)) - const isResponseLegit = await checkIfDDOResponseIsLegit(ddo) + const isResponseLegit = await checkIfDDOResponseIsLegit(ddo, node) if (isResponseLegit) { const ddoInfo: FindDDOResponse = { @@ -898,9 +905,13 @@ export function validateDDOIdentifier(identifier: string): ValidateParams { /** * Checks if the response is legit * @param ddo the DDO + * @param oceanNode the OceanNode instance * @returns validation result */ -async function checkIfDDOResponseIsLegit(ddo: any): Promise { +async function checkIfDDOResponseIsLegit( + ddo: any, + oceanNode: OceanNode +): Promise { const clonedDdo = structuredClone(ddo) const { indexedMetadata } = clonedDdo const updatedDdo = deleteIndexedMetadataIfExists(ddo) @@ -927,8 +938,14 @@ async function checkIfDDOResponseIsLegit(ddo: any): Promise { return false } // 4) check if was deployed by our factory - const blockchain = new Blockchain(network.rpc, chainId, config, network.fallbackRPCs) - const signer = blockchain.getSigner() + const blockchain = oceanNode.getBlockchain(chainId as number) + if (!blockchain) { + CORE_LOGGER.error( + `Blockchain instance not available for chain ${chainId}, cannot confirm validation.` + ) + return false + } + const signer = await blockchain.getSigner() const wasDeployedByUs = await wasNFTDeployedByOurFactory( chainId as number, @@ -942,7 +959,7 @@ async function checkIfDDOResponseIsLegit(ddo: any): Promise { } // 5) check block & events - const networkBlock = await getNetworkHeight(blockchain.getProvider()) + const networkBlock = await getNetworkHeight(await blockchain.getProvider()) if ( !indexedMetadata.event.block || indexedMetadata.event.block < 0 || @@ -960,7 +977,8 @@ async function checkIfDDOResponseIsLegit(ddo: any): Promise { CORE_LOGGER.error(`DDO event missing tx data, cannot confirm transaction`) return false } - const receipt = await blockchain.getProvider().getTransactionReceipt(txId) + const provider = await blockchain.getProvider() + const receipt = await provider.getTransactionReceipt(txId) let foundEvents = false if (receipt) { const { logs } = receipt diff --git a/src/components/core/handler/downloadHandler.ts b/src/components/core/handler/downloadHandler.ts index 7cf559b9f..0fd5d6ade 100644 --- a/src/components/core/handler/downloadHandler.ts +++ b/src/components/core/handler/downloadHandler.ts @@ -2,10 +2,8 @@ import { CommandHandler } from './handler.js' import { MetadataStates, PROTOCOL_COMMANDS } from '../../../utils/constants.js' import { P2PCommandResponse } from '../../../@types/OceanNode.js' import { verifyProviderFees } from '../utils/feesHandler.js' -import { decrypt } from '../../../utils/crypt.js' import { FindDdoHandler } from './ddoHandler.js' import crypto from 'crypto' -import * as ethCrypto from 'eth-crypto' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../../utils/logging/Logger.js' import { validateOrderTransaction } from '../utils/validateOrders.js' import { @@ -16,11 +14,7 @@ import { isERC20Template4Active } from '../../../utils/asset.js' import { Storage } from '../../storage/index.js' -import { - Blockchain, - getConfiguration, - isPolicyServerConfigured -} from '../../../utils/index.js' +import { getConfiguration, isPolicyServerConfigured } from '../../../utils/index.js' import { checkCredentials } from '../../../utils/credentials.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { OceanNode } from '../../../OceanNode.js' @@ -106,13 +100,10 @@ export async function handleDownloadUrlCommand( `attachment;filename=${fileMetadata.name}` if (encryptFile) { // we parse the string into the object again - const encryptedObject = ethCrypto.cipher.parse(task.aes_encrypted_key) - // get the key from configuration - const nodePrivateKey = Buffer.from(config.keys.privateKey.raw).toString('hex') - const decrypted = await ethCrypto.decryptWithPrivateKey( - nodePrivateKey, - encryptedObject - ) + + const decrypted = await node + .getKeyManager() + .ethCryptoDecryptWithPrivateKey(task.aes_encrypted_key) const decryptedPayload = JSON.parse(decrypted) // check signature // const senderAddress = ethCrypto.recover( @@ -266,11 +257,22 @@ export class DownloadHandler extends CommandHandler { // Initialize blockchain early (needed for credential checks with accessList) const config = await getConfiguration() - const { rpc, chainId, fallbackRPCs } = config.supportedNetworks[ddoChainId] + const { chainId } = config.supportedNetworks[ddoChainId] let provider let blockchain + let oceanNode: OceanNode try { - blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) + oceanNode = this.getOceanNode() + blockchain = oceanNode.getBlockchain(chainId) + if (!blockchain) { + return { + stream: null, + status: { + httpStatus: 400, + error: `Download handler: Blockchain instance not available for chain ${chainId}` + } + } + } const { ready, error } = await blockchain.isNetworkReady() if (!ready) { return { @@ -281,7 +283,7 @@ export class DownloadHandler extends CommandHandler { } } } - provider = blockchain.getProvider() + provider = await blockchain.getProvider() } catch (e) { CORE_LOGGER.error('Download JsonRpcProvider ERROR: ' + e.message) return { @@ -292,19 +294,6 @@ export class DownloadHandler extends CommandHandler { } } } - if (!rpc) { - CORE_LOGGER.logMessage( - `Cannot proceed with download. RPC not configured for this chain ${ddo.chainId}`, - true - ) - return { - stream: null, - status: { - httpStatus: 500, - error: `Cannot proceed with download. RPC not configured for this chain ${ddo.chainId}` - } - } - } // check credentials (DDO level) let accessGrantedDDOLevel: boolean @@ -324,7 +313,7 @@ export class DownloadHandler extends CommandHandler { accessGrantedDDOLevel = await checkCredentials( task.consumerAddress, credentials as Credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } if (!accessGrantedDDOLevel) { @@ -365,7 +354,7 @@ export class DownloadHandler extends CommandHandler { accessGrantedServiceLevel = await checkCredentials( task.consumerAddress, service.credentials, - blockchain.getSigner() + await blockchain.getSigner() ) } @@ -391,9 +380,7 @@ export class DownloadHandler extends CommandHandler { // get all compute envs const computeAddrs: string[] = [] - const environments = await this.getOceanNode() - .getC2DEngines() - .fetchEnvironments(ddo.chainId) + const environments = await oceanNode.getC2DEngines().fetchEnvironments(ddo.chainId) for (const env of environments) computeAddrs.push(env.consumerAddress?.toLowerCase()) @@ -435,7 +422,7 @@ export class DownloadHandler extends CommandHandler { service.datatokenAddress, AssetUtils.getServiceIndexById(ddo, task.serviceId), service.timeout, - blockchain.getSigner() + await blockchain.getSigner() ) if (paymentValidation.isValid) { @@ -467,7 +454,7 @@ export class DownloadHandler extends CommandHandler { const confidentialEVM = isConfidentialChainDDO(BigInt(ddo.chainId), service) // check that files is missing and template 4 is active on the chain if (confidentialEVM) { - const signer = blockchain.getSigner() + const signer = await blockchain.getSigner() const isTemplate4 = await isDataTokenTemplate4(service.datatokenAddress, signer) if (!isTemplate4 || !(await isERC20Template4Active(ddo.chainId, signer))) { @@ -507,7 +494,9 @@ export class DownloadHandler extends CommandHandler { const uint8ArrayHex = Uint8Array.from( Buffer.from(sanitizeServiceFiles(filesObject), 'hex') ) - const decryptedUrlBytes = await decrypt(uint8ArrayHex, EncryptMethod.ECIES) + const decryptedUrlBytes = await oceanNode + .getKeyManager() + .decrypt(uint8ArrayHex, EncryptMethod.ECIES) // Convert the decrypted bytes back to a string const decryptedFilesString = Buffer.from(decryptedUrlBytes).toString() decryptedFileData = JSON.parse(decryptedFilesString) diff --git a/src/components/core/handler/encryptHandler.ts b/src/components/core/handler/encryptHandler.ts index 74e5c909d..81af4c4a2 100644 --- a/src/components/core/handler/encryptHandler.ts +++ b/src/components/core/handler/encryptHandler.ts @@ -3,7 +3,6 @@ import { P2PCommandResponse } from '../../../@types/OceanNode.js' import { EncryptCommand, EncryptFileCommand } from '../../../@types/commands.js' import * as base58 from 'base58-js' import { Readable } from 'stream' -import { encrypt } from '../../../utils/crypt.js' import { Storage } from '../../storage/index.js' import { getConfiguration } from '../../../utils/index.js' import { EncryptMethod } from '../../../@types/fileObject.js' @@ -54,6 +53,7 @@ export class EncryptHandler extends CommandHandler { return validationResponse } try { + const oceanNode = this.getOceanNode() // prepare an empty array in case if let blobData: Uint8Array = new Uint8Array() if (task.encoding?.toLowerCase() === 'string') { @@ -65,7 +65,9 @@ export class EncryptHandler extends CommandHandler { blobData = base58.base58_to_binary(task.blob) } // do encrypt magic - const encryptedData = await encrypt(blobData, task.encryptionType) + const encryptedData = await oceanNode + .getKeyManager() + .encrypt(blobData, task.encryptionType) return { stream: Readable.from('0x' + encryptedData.toString('hex')), status: { httpStatus: 200 } @@ -111,21 +113,35 @@ export class EncryptFileHandler extends CommandHandler { return validationResponse } try { + const oceanNode = this.getOceanNode() const config = await getConfiguration() const headers = { 'Content-Type': 'application/octet-stream', - 'X-Encrypted-By': config.keys.peerId.toString(), + 'X-Encrypted-By': oceanNode.getKeyManager().getPeerId().toString(), 'X-Encrypted-Method': task.encryptionType } - let encryptedContent: Buffer + let encryptedContent: Readable if (task.files) { const storage = Storage.getStorageClass(task.files, config) - encryptedContent = await storage.encryptContent(task.encryptionType) + const stream = await storage.getReadableStream() + if (stream.stream) { + encryptedContent = await oceanNode + .getKeyManager() + .encryptStream(stream.stream, task.encryptionType) + } else { + return { + stream: null, + status: { httpStatus: 500, error: 'Cannot fetch files' } + } + } } else if (task.rawData !== null) { - encryptedContent = await encrypt(task.rawData, task.encryptionType) + const cont = await oceanNode + .getKeyManager() + .encrypt(task.rawData, task.encryptionType) + encryptedContent = Readable.from(cont) } return { - stream: Readable.from(encryptedContent), + stream: encryptedContent, status: { httpStatus: 200, headers diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index 762f22c15..096669fea 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -17,7 +17,7 @@ import { ReadableString } from '../../P2P/handlers.js' import { CONNECTION_HISTORY_DELETE_THRESHOLD } from '../../../utils/constants.js' export abstract class BaseHandler implements ICommandHandler { - private nodeInstance: OceanNode + public nodeInstance: OceanNode public constructor(oceanNode: OceanNode) { this.nodeInstance = oceanNode } @@ -32,13 +32,18 @@ export abstract class BaseHandler implements ICommandHandler { } // TODO LOG, implement all handlers - async checkRateLimit(): Promise { + async checkRateLimit(caller: string | string[]): Promise { const requestMap = this.getOceanNode().getRequestMap() const ratePerMinute = (await getConfiguration()).rateLimit - const caller: string | string[] = this.getOceanNode().getRemoteCaller() const requestTime = new Date().getTime() let isOK = true + // If caller is not set, we cannot rate limit - allow the request + if (!caller) { + CORE_LOGGER.debug('No remote caller set, allowing request without rate limiting') + return true + } + // we have to clear this from time to time, so it does not grow forever if (requestMap.size > CONNECTION_HISTORY_DELETE_THRESHOLD) { CORE_LOGGER.info('Request history reached threeshold, cleaning cache...') @@ -152,7 +157,7 @@ export abstract class CommandHandler abstract validate(command: Command): ValidateParams async verifyParamsAndRateLimits(task: Command): Promise { // first check rate limits, if any - if (!(await this.checkRateLimit())) { + if (!(await this.checkRateLimit(task.caller))) { return buildRateLimitReachedResponse() } // then validate the command arguments diff --git a/src/components/core/utils/escrow.ts b/src/components/core/utils/escrow.ts index d30055d06..03c19f350 100644 --- a/src/components/core/utils/escrow.ts +++ b/src/components/core/utils/escrow.ts @@ -6,13 +6,22 @@ import { getOceanArtifactsAdressesByChainId } from '../../../utils/address.js' import { RPCS } from '../../../@types/blockchain.js' import { create256Hash } from '../../../utils/crypt.js' import { sleep } from '../../../utils/util.js' -import { getConfiguration } from '../../../utils/index.js' +import { BlockchainRegistry } from '../../BlockchainRegistry/index.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' + export class Escrow { private networks: RPCS private claimDurationTimeout: number - constructor(supportedNetworks: RPCS, claimDurationTimeout: number) { + private blockchainRegistry: BlockchainRegistry + + constructor( + supportedNetworks: RPCS, + claimDurationTimeout: number, + blockchainRegistry: BlockchainRegistry + ) { this.networks = supportedNetworks this.claimDurationTimeout = claimDurationTimeout + this.blockchainRegistry = blockchainRegistry } // eslint-disable-next-line require-await @@ -26,19 +35,24 @@ export class Escrow { return maxJobDuration + this.claimDurationTimeout } - async getPaymentAmountInWei( - cost: number, - chain: number, - token: string, - existingChain?: Blockchain - ) { - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - let blockchain: Blockchain = existingChain + /** + * Get a Blockchain instance for the given chainId from BlockchainRegistry. + * + * @param chainId - The chain ID to get a Blockchain instance for + * @returns Blockchain instance + * @throws Error if blockchain instance is not available + */ + private getBlockchain(chainId: number): Blockchain { + const blockchain = this.blockchainRegistry.getBlockchain(chainId) if (!blockchain) { - const config = await getConfiguration() - blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) + throw new Error(`Blockchain instance not available for chain ${chainId}`) } - const provider = blockchain.getProvider() + return blockchain + } + + async getPaymentAmountInWei(cost: number, chain: number, token: string) { + const blockchain = this.getBlockchain(chain) + const provider = await blockchain.getProvider() const decimalgBigNumber = await getDatatokenDecimals(token, provider) const decimals = parseInt(decimalgBigNumber.toString()) @@ -49,10 +63,8 @@ export class Escrow { } async getNumberFromWei(wei: string, chain: number, token: string) { - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - const config = await getConfiguration() - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - const provider = blockchain.getProvider() + const blockchain = this.getBlockchain(chain) + const provider = await blockchain.getProvider() const decimals = await getDatatokenDecimals(token, provider) return parseFloat(formatUnits(wei, decimals)) } @@ -67,21 +79,16 @@ export class Escrow { async getUserAvailableFunds( chain: number, payer: string, - token: string, - existingChain?: Blockchain + token: string ): Promise { - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - let blockchain: Blockchain = existingChain - if (!blockchain) { - const config = await getConfiguration() - blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - } - const signer = blockchain.getSigner() - const contract = this.getContract(chainId, signer) + const blockchain = this.getBlockchain(chain) + const signer = await blockchain.getSigner() + const contract = this.getContract(chain, signer) try { const funds = await contract.getUserFunds(payer, token) return funds.available } catch (e) { + CORE_LOGGER.error('Failed to get user available funds: ' + e.message) return null } } @@ -92,14 +99,13 @@ export class Escrow { payer: string, payee: string ): Promise { - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - const config = await getConfiguration() - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - const signer = blockchain.getSigner() - const contract = this.getContract(chainId, signer) + const blockchain = this.getBlockchain(chain) + const signer = await blockchain.getSigner() + const contract = this.getContract(chain, signer) try { return await contract.getLocks(token, payer, payee) } catch (e) { + CORE_LOGGER.error('Failed to get locks: ' + e.message) return null } } @@ -108,20 +114,15 @@ export class Escrow { chain: number, token: string, payer: string, - payee: string, - existingChain?: Blockchain + payee: string ): Promise { - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - let blockchain: Blockchain = existingChain - if (!blockchain) { - const config = await getConfiguration() - blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - } - const signer = blockchain.getSigner() - const contract = this.getContract(chainId, signer) + const blockchain = this.getBlockchain(chain) + const signer = await blockchain.getSigner() + const contract = this.getContract(chain, signer) try { return await contract.getAuthorizations(token, payer, payee) } catch (e) { + CORE_LOGGER.error('Failed to get authorizations: ' + e.message) return null } } @@ -135,14 +136,12 @@ export class Escrow { expiry: BigNumberish ): Promise { const jobId = create256Hash(job) - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - const config = await getConfiguration() - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - const signer = blockchain.getSigner() - const contract = this.getContract(chainId, signer) + const blockchain = this.getBlockchain(chain) + const signer = await blockchain.getSigner() + const contract = this.getContract(chain, signer) if (!contract) throw new Error(`Failed to initialize escrow contract`) - const wei = await this.getPaymentAmountInWei(amount, chain, token, blockchain) - const userBalance = await this.getUserAvailableFunds(chain, payer, token, blockchain) + const wei = await this.getPaymentAmountInWei(amount, chain, token) + const userBalance = await this.getUserAvailableFunds(chain, payer, token) if (BigInt(userBalance.toString()) < BigInt(wei)) { // not enough funds throw new Error(`User ${payer} does not have enough funds`) @@ -153,9 +152,9 @@ export class Escrow { let retries = 2 let auths: EscrowAuthorization[] = [] while (retries > 0) { - auths = await this.getAuthorizations(chain, token, payer, signerAddress, blockchain) + auths = await this.getAuthorizations(chain, token, payer, signerAddress) if (!auths || auths.length !== 1) { - console.log( + CORE_LOGGER.error( `No escrow auths found for: chain=${chain}, token=${token}, payer=${payer}, nodeAddress=${signerAddress}. Found ${ auths?.length || 0 } authorizations. ${retries > 0 ? 'Retrying..' : ''}` @@ -197,7 +196,7 @@ export class Escrow { const tx = await contract.createLock(jobId, token, payer, wei, expiry, gasOptions) return tx.hash } catch (e) { - console.log(e) + CORE_LOGGER.error('Failed to create lock: ' + e.message) throw new Error(String(e.message)) } } @@ -210,12 +209,10 @@ export class Escrow { amount: number, proof: string ): Promise { - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - const config = await getConfiguration() - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - const signer = blockchain.getSigner() - const contract = this.getContract(chainId, signer) - const wei = await this.getPaymentAmountInWei(amount, chain, token, blockchain) + const blockchain = this.getBlockchain(chain) + const signer = await blockchain.getSigner() + const contract = this.getContract(chain, signer) + const wei = await this.getPaymentAmountInWei(amount, chain, token) const jobId = create256Hash(job) if (!contract) return null try { @@ -243,6 +240,7 @@ export class Escrow { } return null } catch (e) { + CORE_LOGGER.error('Failed to claim lock: ' + e.message) throw new Error(String(e.message)) } } @@ -253,12 +251,10 @@ export class Escrow { token: string, payer: string ): Promise { - const { rpc, chainId, fallbackRPCs } = this.networks[chain] - const config = await getConfiguration() - const blockchain = new Blockchain(rpc, chainId, config, fallbackRPCs) - const signer = blockchain.getSigner() + const blockchain = this.getBlockchain(chain) + const signer = await blockchain.getSigner() const jobId = create256Hash(job) - const contract = this.getContract(chainId, signer) + const contract = this.getContract(chain, signer) if (!contract) return null try { @@ -285,6 +281,7 @@ export class Escrow { } return null } catch (e) { + CORE_LOGGER.error('Failed to cancel expired locks: ' + e.message) throw new Error(String(e.message)) } } diff --git a/src/components/core/utils/feesHandler.ts b/src/components/core/utils/feesHandler.ts index 32fdb5af0..b351e5cf8 100644 --- a/src/components/core/utils/feesHandler.ts +++ b/src/components/core/utils/feesHandler.ts @@ -1,12 +1,13 @@ import type { ComputeResourcesPricingInfo } from '../../../@types/C2D/C2D.js' import { - JsonRpcApiProvider, + FallbackProvider, ethers, Interface, BigNumberish, parseUnits, ZeroAddress } from 'ethers' +import { OceanNode } from '../../../OceanNode.js' import { FeeTokens, ProviderFeeData, @@ -133,7 +134,7 @@ export async function createProviderFee( export async function verifyProviderFees( txId: string, userAddress: string, - provider: JsonRpcApiProvider, + provider: FallbackProvider, service: Service ): Promise { /* given a transaction, check if there is a valid provider fee event @@ -450,17 +451,14 @@ export async function checkFee( * @param chainId the chain id (not used now) * @returns the wallet */ +// eslint-disable-next-line require-await export async function getProviderWallet(chainId?: string): Promise { - return new ethers.Wallet( - Buffer.from((await getConfiguration()).keys.privateKey.raw).toString('hex') - ) + const oceanNode = OceanNode.getInstance() + const keyManager = oceanNode.getKeyManager() + return keyManager.getEthWallet() } export async function getProviderWalletAddress(): Promise { - return (await getProviderWallet()).address -} - -export async function getProviderKey(): Promise { - return Buffer.from((await getConfiguration()).keys.privateKey.raw).toString('hex') + return (await this.getProviderWallet()).address } /** diff --git a/src/components/core/utils/findDdoHandler.ts b/src/components/core/utils/findDdoHandler.ts index f83eddf64..02693903f 100644 --- a/src/components/core/utils/findDdoHandler.ts +++ b/src/components/core/utils/findDdoHandler.ts @@ -5,7 +5,7 @@ import { FindDDOResponse } from '../../../@types/index.js' import { Service } from '@oceanprotocol/ddo-js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { OceanNode } from '../../../OceanNode.js' -import { getConfiguration, hasP2PInterface } from '../../../utils/config.js' +import { hasP2PInterface } from '../../../utils/config.js' /** * Check if the specified ddo is cached and if the cached version is recent enough @@ -73,7 +73,7 @@ export async function findDDOLocally( // node has ddo const p2pNode: OceanP2P = node.getP2PNode() if (!p2pNode || !hasP2PInterface) { - const peerId: string = await (await getConfiguration()).keys.peerId.toString() + const peerId: string = node.getKeyManager().getPeerId().toString() return { id: ddo.id, lastUpdateTx: ddo.event.tx, diff --git a/src/components/core/utils/statusHandler.ts b/src/components/core/utils/statusHandler.ts index 6d69b2e29..c69d2c069 100644 --- a/src/components/core/utils/statusHandler.ts +++ b/src/components/core/utils/statusHandler.ts @@ -115,12 +115,17 @@ export async function status( // no previous status? if (!nodeStatus) { - const publicKeyHex = Buffer.from(config.keys.publicKey).toString('hex') + const publicKey = oceanNode.getKeyManager().getPublicKey() + const publicKeyHex = Buffer.from(publicKey).toString('hex') + nodeStatus = { - id: nodeId && nodeId !== undefined ? nodeId : config.keys.peerId.toString(), // get current node ID + id: + nodeId && nodeId !== undefined + ? nodeId + : oceanNode.getKeyManager().getPeerId().toString(), // get current node ID publicKey: publicKeyHex, friendlyName: new HumanHasher().humanize(publicKeyHex), - address: config.keys.ethAddress, + address: oceanNode.getKeyManager().getEthAddress(), version: process.env.npm_package_version, http: config.hasHttp, p2p: config.hasP2P, diff --git a/src/components/core/utils/validateOrders.ts b/src/components/core/utils/validateOrders.ts index f23eb1e89..8210d150e 100644 --- a/src/components/core/utils/validateOrders.ts +++ b/src/components/core/utils/validateOrders.ts @@ -1,10 +1,4 @@ -import { - JsonRpcApiProvider, - Contract, - Interface, - TransactionReceipt, - Signer -} from 'ethers' +import { Contract, Interface, TransactionReceipt, Signer, FallbackProvider } from 'ethers' import { fetchEventFromTransaction } from '../../../utils/util.js' import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20TemplateEnterprise.sol/ERC20TemplateEnterprise.json' with { type: 'json' } import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC721Template.sol/ERC721Template.json' with { type: 'json' } @@ -20,7 +14,7 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) export async function fetchTransactionReceipt( txId: string, - provider: JsonRpcApiProvider, + provider: FallbackProvider, retries: number = 2 ): Promise { while (retries > 0) { @@ -45,7 +39,7 @@ export async function fetchTransactionReceipt( export async function validateOrderTransaction( txId: string, userAddress: string, - provider: JsonRpcApiProvider, + provider: FallbackProvider, dataNftAddress: string, datatokenAddress: string, serviceIndex: number, diff --git a/src/components/httpRoutes/adminConfig.ts b/src/components/httpRoutes/adminConfig.ts index c00112367..4a3c0d071 100644 --- a/src/components/httpRoutes/adminConfig.ts +++ b/src/components/httpRoutes/adminConfig.ts @@ -16,7 +16,8 @@ adminConfigRoutes.get('/api/admin/config', express.json(), async (req, res) => { command: PROTOCOL_COMMANDS.FETCH_CONFIG, expiryTimestamp, signature, - address + address, + caller: req.caller }) if (response.status.httpStatus === 200) { @@ -41,7 +42,8 @@ adminConfigRoutes.post('/api/admin/config/update', express.json(), async (req, r expiryTimestamp, signature, config, - address + address, + caller: req.caller }) if (response.status.httpStatus === 200) { diff --git a/src/components/httpRoutes/aquarius.ts b/src/components/httpRoutes/aquarius.ts index e3c3eebb6..d440413fa 100644 --- a/src/components/httpRoutes/aquarius.ts +++ b/src/components/httpRoutes/aquarius.ts @@ -77,7 +77,8 @@ aquariusRoutes.post( const result = await new QueryHandler(req.oceanNode).handle({ query: transformedQuery, - command: PROTOCOL_COMMANDS.QUERY + command: PROTOCOL_COMMANDS.QUERY, + caller: req.caller }) if (result.stream) { const queryResult = JSON.parse(await streamToString(result.stream as Readable)) @@ -103,7 +104,8 @@ aquariusRoutes.get(`${AQUARIUS_API_BASE_PATH}/state/ddo`, async (req, res) => { const queryDdoState: QueryCommand = { query: queryStrategy.buildQuery(did, nft, txId), - command: PROTOCOL_COMMANDS.QUERY + command: PROTOCOL_COMMANDS.QUERY, + caller: req.caller } if (!Object.keys(queryDdoState.query).length) { @@ -179,7 +181,8 @@ aquariusRoutes.post(`${AQUARIUS_API_BASE_PATH}/assets/ddo/validate`, async (req, nonce, signature, message: ddo.id + nonce, - command: PROTOCOL_COMMANDS.VALIDATE_DDO + command: PROTOCOL_COMMANDS.VALIDATE_DDO, + caller: req.caller }) if (result.stream) { diff --git a/src/components/httpRoutes/auth.ts b/src/components/httpRoutes/auth.ts index 68a44860d..389954b4a 100644 --- a/src/components/httpRoutes/auth.ts +++ b/src/components/httpRoutes/auth.ts @@ -27,7 +27,8 @@ authRoutes.post( address, nonce, validUntil, - chainId + chainId, + caller: req.caller }) if (response.status.error) { @@ -62,7 +63,8 @@ authRoutes.post( address, nonce, token, - chainId + chainId, + caller: req.caller }) if (response.status.error) { diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index 0c4a1f6ae..c7137b89a 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -42,7 +42,8 @@ computeRoutes.get(`${SERVICES_API_BASE_PATH}/computeEnvironments`, async (req, r const getEnvironmentsTask = { command: PROTOCOL_COMMANDS.COMPUTE_GET_ENVIRONMENTS, chainId: parseInt(req.query.chainId as string) || null, - node: (req.query.node as string) || null + node: (req.query.node as string) || null, + caller: req.caller } const response = await new ComputeGetEnvironmentsHandler(req.oceanNode).handle( getEnvironmentsTask @@ -81,7 +82,8 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { metadata: req.body.metadata || null, authorization: req.headers?.authorization, additionalViewers: (req.body.additionalViewers as unknown as string[]) || null, - queueMaxWaitTime: req.body.queueMaxWaitTime || 0 + queueMaxWaitTime: req.body.queueMaxWaitTime || 0, + caller: req.caller } if (req.body.output) { startComputeTask.output = req.body.output as ComputeOutput @@ -127,7 +129,8 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/freeCompute`, async (req, res) => metadata: req.body.metadata || null, authorization: req.headers?.authorization, additionalViewers: (req.body.additionalViewers as unknown as string[]) || null, - queueMaxWaitTime: req.body.queueMaxWaitTime || 0 + queueMaxWaitTime: req.body.queueMaxWaitTime || 0, + caller: req.caller } if (req.body.output) { startComputeTask.output = req.body.output as ComputeOutput @@ -167,7 +170,8 @@ computeRoutes.put(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { nonce: (req.query.nonce as string) || null, jobId: (req.query.jobId as string) || null, agreementId: (req.query.agreementId as string) || null, - authorization: req.headers?.authorization || null + authorization: req.headers?.authorization || null, + caller: req.caller } const response = await new ComputeStopHandler(req.oceanNode).handle(stopComputeTask) const jobs = await streamToObject(response.stream as Readable) @@ -190,7 +194,8 @@ computeRoutes.get(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { node: (req.query.node as string) || null, consumerAddress: (req.query.consumerAddress as string) || null, jobId: (req.query.jobId as string) || null, - agreementId: (req.query.agreementId as string) || null + agreementId: (req.query.agreementId as string) || null, + caller: req.caller } const response = await new ComputeGetStatusHandler(req.oceanNode).handle( statusComputeTask @@ -218,7 +223,8 @@ computeRoutes.get(`${SERVICES_API_BASE_PATH}/computeResult`, async (req, res) => jobId: (req.query.jobId as string) || null, signature: (req.query.signature as string) || null, nonce: (req.query.nonce as string) || null, - authorization: req.headers?.authorization + authorization: req.headers?.authorization, + caller: req.caller } const response = await new ComputeGetResultHandler(req.oceanNode).handle( @@ -254,7 +260,8 @@ computeRoutes.get(`${SERVICES_API_BASE_PATH}/computeStreamableLogs`, async (req, jobId: (req.query.jobId as string) || null, signature: (req.query.signature as string) || null, nonce: (req.query.nonce as string) || null, - authorization: req.headers?.authorization + authorization: req.headers?.authorization, + caller: req.caller } const response = await new ComputeGetStreamableLogsHandler(req.oceanNode).handle( diff --git a/src/components/httpRoutes/fileInfo.ts b/src/components/httpRoutes/fileInfo.ts index 7c3da3cd9..14f96f2a1 100644 --- a/src/components/httpRoutes/fileInfo.ts +++ b/src/components/httpRoutes/fileInfo.ts @@ -51,7 +51,8 @@ fileInfoRoute.post( fileInfoTask = { command: PROTOCOL_COMMANDS.FILE_INFO, did: fileInfoReq.did, - serviceId: fileInfoReq.serviceId + serviceId: fileInfoReq.serviceId, + caller: req.caller } } else if (fileInfoReq.type === 'url' && fileInfoReq.url) { fileObject = { @@ -62,7 +63,8 @@ fileInfoRoute.post( fileInfoTask = { command: PROTOCOL_COMMANDS.FILE_INFO, file: fileObject, - type: fileObject.type as FileObjectType + type: fileObject.type as FileObjectType, + caller: req.caller } } else if (fileInfoReq.type === 'ipfs' && fileInfoReq.hash) { fileObject = { @@ -73,7 +75,8 @@ fileInfoRoute.post( fileInfoTask = { command: PROTOCOL_COMMANDS.FILE_INFO, file: fileObject, - type: fileObject.type as FileObjectType + type: fileObject.type as FileObjectType, + caller: req.caller } } else if (fileInfoReq.type === 'arweave' && fileInfoReq.transactionId) { fileObject = { @@ -84,7 +87,8 @@ fileInfoRoute.post( fileInfoTask = { command: PROTOCOL_COMMANDS.FILE_INFO, file: fileObject, - type: fileObject.type as FileObjectType + type: fileObject.type as FileObjectType, + caller: req.caller } } const response = await new FileInfoHandler(req.oceanNode).handle(fileInfoTask) diff --git a/src/components/httpRoutes/getOceanPeers.ts b/src/components/httpRoutes/getOceanPeers.ts index cc81a7391..66366b5c0 100644 --- a/src/components/httpRoutes/getOceanPeers.ts +++ b/src/components/httpRoutes/getOceanPeers.ts @@ -15,7 +15,8 @@ p2pRoutes.get( async (req: Request, res: Response): Promise => { const node = req.oceanNode const result = await new GetP2PNetworkStatsHandler(node).handle({ - command: PROTOCOL_COMMANDS.GET_P2P_NETWORK_STATS + command: PROTOCOL_COMMANDS.GET_P2P_NETWORK_STATS, + caller: req.caller }) if (result.stream) { const validationResult = JSON.parse(await streamToString(result.stream as Readable)) @@ -38,7 +39,8 @@ p2pRoutes.get( const result = await new FindPeerHandler(node).handle({ command: PROTOCOL_COMMANDS.FIND_PEER, peerId: req.query.peerId as string, - timeout: req.query.timeout as string + timeout: req.query.timeout as string, + caller: req.caller }) if (result.stream) { const validationResult = JSON.parse(await streamToString(result.stream as Readable)) @@ -53,7 +55,8 @@ export const getP2PPeersRoute = express.Router() p2pRoutes.get('/getP2PPeers', async (req: Request, res: Response): Promise => { const node = req.oceanNode const result = await new GetP2PPeersHandler(node).handle({ - command: PROTOCOL_COMMANDS.GET_P2P_PEERS + command: PROTOCOL_COMMANDS.GET_P2P_PEERS, + caller: req.caller }) if (result.stream) { const validationResult = JSON.parse(await streamToString(result.stream as Readable)) @@ -75,7 +78,8 @@ p2pRoutes.get( const node = req.oceanNode const result = await new GetP2PPeerHandler(node).handle({ command: PROTOCOL_COMMANDS.GET_P2P_PEER, - peerId: req.query.peerId as string + peerId: req.query.peerId as string, + caller: req.caller }) if (result.stream) { const validationResult = JSON.parse(await streamToString(result.stream as Readable)) diff --git a/src/components/httpRoutes/policyServer.ts b/src/components/httpRoutes/policyServer.ts index df77a813e..8b2bf3350 100644 --- a/src/components/httpRoutes/policyServer.ts +++ b/src/components/httpRoutes/policyServer.ts @@ -20,7 +20,8 @@ PolicyServerPassthroughRoute.post( try { const response = await new PolicyServerPassthroughHandler(req.oceanNode).handle({ command: PROTOCOL_COMMANDS.POLICY_SERVER_PASSTHROUGH, - policyServerPassthrough: req.body.policyServerPassthrough + policyServerPassthrough: req.body.policyServerPassthrough, + caller: req.caller }) if (response.stream) { res.status(response.status.httpStatus) @@ -52,7 +53,8 @@ PolicyServerPassthroughRoute.post( documentId: req.body.documentId, serviceId: req.body.serviceId, consumerAddress: req.body.consumerAddress, - policyServer: req.body.policyServer + policyServer: req.body.policyServer, + caller: req.caller }) if (response.stream) { res.status(response.status.httpStatus) diff --git a/src/components/httpRoutes/provider.ts b/src/components/httpRoutes/provider.ts index 7cc09f687..bc45661af 100644 --- a/src/components/httpRoutes/provider.ts +++ b/src/components/httpRoutes/provider.ts @@ -20,7 +20,8 @@ providerRoutes.post(`${SERVICES_API_BASE_PATH}/decrypt`, async (req, res) => { try { const result = await new DecryptDdoHandler(req.oceanNode).handle({ ...req.body, - command: PROTOCOL_COMMANDS.DECRYPT_DDO + command: PROTOCOL_COMMANDS.DECRYPT_DDO, + caller: req.caller }) if (result.stream) { const decryptedData = await streamToString(result.stream as Readable) @@ -46,7 +47,8 @@ providerRoutes.post(`${SERVICES_API_BASE_PATH}/encrypt`, async (req, res) => { blob: data, encoding: 'string', encryptionType: EncryptMethod.ECIES, - command: PROTOCOL_COMMANDS.ENCRYPT + command: PROTOCOL_COMMANDS.ENCRYPT, + caller: req.caller }) if (result.stream) { const encryptedData = await streamToString(result.stream as Readable) @@ -97,7 +99,8 @@ providerRoutes.post(`${SERVICES_API_BASE_PATH}/encryptFile`, async (req, res) => const result = await new EncryptFileHandler(req.oceanNode).handle({ rawData: input, encryptionType: encryptMethod, - command: PROTOCOL_COMMANDS.ENCRYPT_FILE + command: PROTOCOL_COMMANDS.ENCRYPT_FILE, + caller: req.caller }) return result } @@ -112,7 +115,8 @@ providerRoutes.post(`${SERVICES_API_BASE_PATH}/encryptFile`, async (req, res) => result = await new EncryptFileHandler(req.oceanNode).handle({ files: req.body as BaseFileObject, encryptionType: encryptMethod, - command: PROTOCOL_COMMANDS.ENCRYPT_FILE + command: PROTOCOL_COMMANDS.ENCRYPT_FILE, + caller: req.caller }) return await writeResponse(result, encryptMethod) // raw data on body @@ -153,7 +157,8 @@ providerRoutes.get(`${SERVICES_API_BASE_PATH}/initialize`, async (req, res) => { serviceId: (req.query.serviceId as string) || null, consumerAddress: (req.query.consumerAddress as string) || null, validUntil: parseInt(req.query.validUntil as string) || null, - policyServer: (req.query.policyServer as any) || null + policyServer: (req.query.policyServer as any) || null, + caller: req.caller }) if (result.stream) { const initializeREsponse = await streamToObject(result.stream as Readable) @@ -234,7 +239,8 @@ providerRoutes.get( command: PROTOCOL_COMMANDS.DOWNLOAD, policyServer: (req.query.policyServer as any) || null, authorization: authorization as string, - userData: parsedUserData + userData: parsedUserData, + caller: req.caller } const response = await new DownloadHandler(req.oceanNode).handle(downloadTask) diff --git a/src/components/httpRoutes/rootEndpoint.ts b/src/components/httpRoutes/rootEndpoint.ts index 8322b9b21..4f3a60684 100644 --- a/src/components/httpRoutes/rootEndpoint.ts +++ b/src/components/httpRoutes/rootEndpoint.ts @@ -9,10 +9,11 @@ rootEndpointRoutes.get('/', async (req, res) => { if (!config.supportedNetworks) { HTTP_LOGGER.warn(`Supported networks not defined`) } + const keyManager = req.oceanNode.getKeyManager() res.json({ - nodeId: config.keys.peerId, + nodeId: keyManager.getPeerId().toString(), chainIds: config.supportedNetworks ? Object.keys(config.supportedNetworks) : [], - providerAddress: config.keys.ethAddress, + providerAddress: keyManager.getEthAddress(), serviceEndpoints: getAllServiceEndpoints(), software: 'Ocean-Node', version: '0.0.1' diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index 173c342f1..6842da24e 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -12,8 +12,7 @@ import { OceanNodeConfig } from '../../@types/OceanNode.js' import { fetchFileMetadata } from '../../utils/asset.js' import axios from 'axios' import urlJoin from 'url-join' -import { encrypt as encryptData, decrypt as decryptData } from '../../utils/crypt.js' -import { Readable } from 'stream' + import { CORE_LOGGER } from '../../utils/logging/common.js' export abstract class Storage { @@ -35,7 +34,6 @@ export abstract class Storage { forceChecksum: boolean ): Promise - abstract encryptContent(encryptionType: 'AES' | 'ECIES'): Promise abstract isFilePath(): boolean getFile(): any { @@ -116,58 +114,6 @@ export abstract class Storage { return response } - async encrypt(encryptionType: EncryptMethod = EncryptMethod.AES) { - const readableStream = await this.getReadableStream() - - // Convert the readable stream to a buffer - const chunks: Buffer[] = [] - for await (const chunk of readableStream.stream) { - chunks.push(chunk) - } - const buffer = Buffer.concat(chunks) - - // Encrypt the buffer using the encrypt function - const encryptedBuffer = await encryptData(new Uint8Array(buffer), encryptionType) - - // Convert the encrypted buffer back into a stream - const encryptedStream = Readable.from(encryptedBuffer) - - return { - ...readableStream, - stream: encryptedStream - } - } - - async decrypt() { - const { keys } = this.config - const nodeId = keys.peerId.toString() - - if (!this.canDecrypt(nodeId)) { - throw new Error('Node is not authorized to decrypt this file') - } - - const { encryptMethod } = this.file - const readableStream = await this.getReadableStream() - - // Convert the readable stream to a buffer - const chunks: Buffer[] = [] - for await (const chunk of readableStream.stream) { - chunks.push(chunk) - } - const buffer = Buffer.concat(chunks) - - // Decrypt the buffer using your existing function - const decryptedBuffer = await decryptData(new Uint8Array(buffer), encryptMethod) - - // Convert the decrypted buffer back into a stream - const decryptedStream = Readable.from(decryptedBuffer) - - return { - ...readableStream, - stream: decryptedStream - } - } - isEncrypted(): boolean { if ( this.file.encryptedBy && @@ -265,19 +211,6 @@ export class UrlStorage extends Storage { encryptMethod: fileObject.encryptMethod } } - - async encryptContent( - encryptionType: EncryptMethod.AES | EncryptMethod.ECIES - ): Promise { - const file = this.getFile() - const response = await axios({ - url: file.url, - method: file.method || 'get', - headers: file.headers, - timeout: 30000 - }) - return await encryptData(response.data, encryptionType) - } } export class ArweaveStorage extends Storage { @@ -345,17 +278,6 @@ export class ArweaveStorage extends Storage { encryptMethod: fileObject.encryptMethod } } - - async encryptContent( - encryptionType: EncryptMethod.AES | EncryptMethod.ECIES - ): Promise { - const file = this.getFile() - const response = await axios({ - url: urlJoin(this.config.arweaveGateway, file.transactionId), - method: 'get' - }) - return await encryptData(response.data, encryptionType) - } } export class IpfsStorage extends Storage { @@ -417,15 +339,4 @@ export class IpfsStorage extends Storage { encryptMethod: fileObject.encryptMethod } } - - async encryptContent( - encryptionType: EncryptMethod.AES | EncryptMethod.ECIES - ): Promise { - const file = this.getFile() - const response = await axios({ - url: file.hash, - method: 'get' - }) - return await encryptData(response.data, encryptionType) - } } diff --git a/src/index.ts b/src/index.ts index ba2a094a9..532dc363d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,8 @@ import { OceanIndexer } from './components/Indexer/index.js' import { Database } from './components/database/index.js' import express, { Express } from 'express' import { OceanNode } from './OceanNode.js' +import { KeyManager } from './components/KeyManager/index.js' +import { BlockchainRegistry } from './components/BlockchainRegistry/index.js' import { httpRoutes } from './components/httpRoutes/index.js' import { getConfiguration, @@ -34,6 +36,7 @@ declare global { // eslint-disable-next-line no-unused-vars interface Request { oceanNode: OceanNode + caller?: string | string[] } } } @@ -66,7 +69,6 @@ const isStartup: boolean = true // this is to avoid too much verbose logging, cause we're calling getConfig() from many parts // and we are always running though the same process.env checks // (we must start accessing the config from the OceanNode class only once we refactor) -console.log('\n\n\n\n') OCEAN_NODE_LOGGER.logMessageWithEmoji( '[ Starting Ocean Node ]', true, @@ -102,16 +104,21 @@ if (!hasValidDBConfiguration(config.dbConfig)) { ) } +// Create KeyManager and BlockchainRegistry +// KeyManager will determine provider type from config.keys.type and initialize in constructor +const keyManager = new KeyManager(config) +const blockchainRegistry = new BlockchainRegistry(keyManager, config) + if (config.hasP2P) { if (dbconn) { - node = new OceanP2P(config, dbconn) + node = new OceanP2P(config, keyManager, dbconn) } else { - node = new OceanP2P(config) + node = new OceanP2P(config, keyManager) } await node.start() } if (config.hasIndexer && dbconn) { - indexer = new OceanIndexer(dbconn, config.indexingNetworks) + indexer = new OceanIndexer(dbconn, config.indexingNetworks, blockchainRegistry) // if we set this var // it also loads initial data (useful for testing, or we might actually want to have a bootstrap list) // store and advertise DDOs @@ -130,7 +137,16 @@ if (dbconn) { } // Singleton instance across application -const oceanNode = OceanNode.getInstance(config, dbconn, node, provider, indexer) +const oceanNode = OceanNode.getInstance( + config, + + dbconn, + node, + provider, + indexer, + keyManager, + blockchainRegistry +) oceanNode.addC2DEngines() function removeExtraSlashes(req: any, res: any, next: any) { @@ -168,7 +184,7 @@ if (config.hasHttp) { } app.use(requestValidator, (req, res, next) => { - oceanNode.setRemoteCaller(req.headers['x-forwarded-for'] || req.socket.remoteAddress) + req.caller = req.headers['x-forwarded-for'] || req.socket.remoteAddress req.oceanNode = oceanNode next() }) diff --git a/src/test/integration/accessLists.test.ts b/src/test/integration/accessLists.test.ts index 23d02cdb1..fe901f65e 100644 --- a/src/test/integration/accessLists.test.ts +++ b/src/test/integration/accessLists.test.ts @@ -7,6 +7,7 @@ import { TEST_ENV_CONFIG_FILE } from '../utils/utils.js' import { JsonRpcProvider, Signer } from 'ethers' +import { BlockchainRegistry } from '../../components/BlockchainRegistry/index.js' import { Blockchain } from '../../utils/blockchain.js' import { RPCS, SupportedNetwork } from '../../@types/blockchain.js' import { DEVELOPMENT_CHAIN_ID } from '../../utils/address.js' @@ -17,6 +18,7 @@ import { homedir } from 'os' import { getConfiguration } from '../../utils/config.js' import { assert, expect } from 'chai' import { checkAddressOnAccessList } from '../../utils/accessList.js' +import { KeyManager } from '../../components/KeyManager/index.js' describe('Should deploy some accessLists before all other tests.', () => { let config: OceanNodeConfig @@ -46,9 +48,11 @@ describe('Should deploy some accessLists before all other tests.', () => { const rpcs: RPCS = config.supportedNetworks const chain: SupportedNetwork = rpcs[String(DEVELOPMENT_CHAIN_ID)] - blockchain = new Blockchain(chain.rpc, chain.chainId, config, chain.fallbackRPCs) + const keyManager = new KeyManager(config) + const blockchains = new BlockchainRegistry(keyManager, config) + blockchain = blockchains.getBlockchain(chain.chainId) - owner = blockchain.getSigner() + owner = await blockchain.getSigner() // ENVIRONMENT_VARIABLES.AUTHORIZED_PUBLISHERS_LIST const accessListPublishers = await deployAndGetAccessListConfig( diff --git a/src/test/integration/algorithmsAccess.test.ts b/src/test/integration/algorithmsAccess.test.ts index d72c3b11c..d754fcf61 100644 --- a/src/test/integration/algorithmsAccess.test.ts +++ b/src/test/integration/algorithmsAccess.test.ts @@ -110,7 +110,11 @@ describe('Trusted algorithms Flow', () => { config = await getConfiguration(true) dbconn = await Database.init(config.dbConfig) oceanNode = await OceanNode.getInstance(config, dbconn, null, null, null) - indexer = new OceanIndexer(dbconn, config.indexingNetworks) + indexer = new OceanIndexer( + dbconn, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) oceanNode.addC2DEngines() diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index c60ff4d93..96278058b 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -67,7 +67,6 @@ import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templat import OceanToken from '@oceanprotocol/contracts/artifacts/contracts/utils/OceanToken.sol/OceanToken.json' with { type: 'json' } import EscrowJson from '@oceanprotocol/contracts/artifacts/contracts/escrow/Escrow.sol/Escrow.json' with { type: 'json' } import { createHash } from 'crypto' -import { encrypt } from '../../utils/crypt.js' import { EncryptMethod } from '../../@types/fileObject.js' import { getAlgoChecksums, @@ -158,8 +157,21 @@ describe('Compute', () => { ) config = await getConfiguration(true) dbconn = await Database.init(config.dbConfig) - oceanNode = await OceanNode.getInstance(config, dbconn, null, null, null, true) - indexer = new OceanIndexer(dbconn, config.indexingNetworks) + oceanNode = await OceanNode.getInstance( + config, + dbconn, + null, + null, + null, + null, + null, + true + ) + indexer = new OceanIndexer( + dbconn, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) oceanNode.addC2DEngines() @@ -1199,7 +1211,9 @@ describe('Compute', () => { ] } const filesData = Uint8Array.from(Buffer.from(JSON.stringify(files))) - algoDDO.services[0].files = await encrypt(filesData, EncryptMethod.ECIES) + algoDDO.services[0].files = await oceanNode + .getKeyManager() + .encrypt(filesData, EncryptMethod.ECIES) const metadata = hexlify(Buffer.from(JSON.stringify(algoDDO))) const hash = createHash('sha256').update(metadata).digest('hex') @@ -1269,7 +1283,9 @@ describe('Compute', () => { ] } const filesData = Uint8Array.from(Buffer.from(JSON.stringify(files))) - datasetDDO.services[0].files = await encrypt(filesData, EncryptMethod.ECIES) + datasetDDO.services[0].files = await oceanNode + .getKeyManager() + .encrypt(filesData, EncryptMethod.ECIES) datasetDDO.services[0].compute = { allowRawAlgorithm: false, @@ -1517,8 +1533,21 @@ describe('Compute Access Restrictions', () => { ) config = await getConfiguration(true) dbconn = await Database.init(config.dbConfig) - oceanNode = await OceanNode.getInstance(config, dbconn, null, null, null, true) - const indexer = new OceanIndexer(dbconn, config.indexingNetworks) + oceanNode = await OceanNode.getInstance( + config, + dbconn, + null, + null, + null, + null, + null, + true + ) + const indexer = new OceanIndexer( + dbconn, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) oceanNode.addC2DEngines() @@ -1698,8 +1727,21 @@ describe('Compute Access Restrictions', () => { ) config = await getConfiguration(true) dbconn = await Database.init(config.dbConfig) - oceanNode = await OceanNode.getInstance(config, dbconn, null, null, null, true) - const indexer = new OceanIndexer(dbconn, config.indexingNetworks) + oceanNode = await OceanNode.getInstance( + config, + dbconn, + null, + null, + null, + null, + null, + true + ) + const indexer = new OceanIndexer( + dbconn, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) oceanNode.addC2DEngines() diff --git a/src/test/integration/configDatabase.test.ts b/src/test/integration/configDatabase.test.ts index 97060d327..2a6af182f 100644 --- a/src/test/integration/configDatabase.test.ts +++ b/src/test/integration/configDatabase.test.ts @@ -1,4 +1,5 @@ import { Database } from '../../components/database/index.js' +import { OceanNode } from '../../OceanNode.js' import { expect, assert } from 'chai' import { OceanIndexer } from '../../components/Indexer/index.js' import { @@ -9,9 +10,11 @@ import { tearDownEnvironment, TEST_ENV_CONFIG_FILE } from '../utils/utils.js' +import { ENVIRONMENT_VARIABLES, getConfiguration } from '../../utils/index.js' import { SQLLiteConfigDatabase } from '../../components/database/SQLLiteConfigDatabase.js' import { DB_TYPES } from '../../utils/constants.js' import { OceanNodeDBConfig } from '../../@types/OceanNode.js' +import { homedir } from 'os' const versionConfig: OceanNodeDBConfig = { url: 'http://localhost:8108/test-version?apiKey=xyz', @@ -31,10 +34,27 @@ describe('Config Database', () => { before(async () => { database = await Database.init(versionConfig) - + // override and save configuration (always before calling getConfig()) previousConfiguration = await setupEnvironment( TEST_ENV_CONFIG_FILE, - buildEnvOverrideConfig([], []) + buildEnvOverrideConfig( + [ + ENVIRONMENT_VARIABLES.RPCS, + ENVIRONMENT_VARIABLES.INDEXER_NETWORKS, + ENVIRONMENT_VARIABLES.PRIVATE_KEY, + ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS, + ENVIRONMENT_VARIABLES.ALLOWED_ADMINS, + ENVIRONMENT_VARIABLES.ADDRESS_FILE + ], + [ + JSON.stringify(getMockSupportedNetworks()), + JSON.stringify([8996]), + '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58', + JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']), + JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']), + `${homedir}/.ocean/ocean-contracts/artifacts/address.json` + ] + ) ) it('should have null version initially', async () => { @@ -42,7 +62,13 @@ describe('Config Database', () => { assert(initialVersionNull.value === null, 'Initial version should be null') }) - oceanIndexer = new OceanIndexer(database, getMockSupportedNetworks()) + const oceanNode = await OceanNode.getInstance(await getConfiguration(true), database) + oceanIndexer = new OceanIndexer( + database, + getMockSupportedNetworks(), + oceanNode.blockchainRegistry + ) + oceanNode.addIndexer(oceanIndexer) }) it('check version DB instance of SQL Lite', () => { diff --git a/src/test/integration/credentials.test.ts b/src/test/integration/credentials.test.ts index 00c661a21..fe76108bc 100644 --- a/src/test/integration/credentials.test.ts +++ b/src/test/integration/credentials.test.ts @@ -140,13 +140,17 @@ describe('[Credentials Flow] - Should run a complete node flow.', () => { config = await getConfiguration(true) // Force reload the configuration const database = await Database.init(config.dbConfig) oceanNode = OceanNode.getInstance(config, database) - const indexer = new OceanIndexer(database, config.indexingNetworks) + const indexer = new OceanIndexer( + database, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) await oceanNode.addC2DEngines() const rpcs: RPCS = config.supportedNetworks const chain: SupportedNetwork = rpcs[String(DEVELOPMENT_CHAIN_ID)] - blockchain = new Blockchain(chain.rpc, chain.chainId, config, chain.fallbackRPCs) + blockchain = oceanNode.blockchainRegistry.getBlockchain(chain.chainId) consumerAccounts = [ (await provider.getSigner(1)) as Signer, @@ -163,7 +167,7 @@ describe('[Credentials Flow] - Should run a complete node flow.', () => { networkArtifacts = getOceanArtifactsAdresses().development } - signer = blockchain.getSigner() + signer = await blockchain.getSigner() const txAddress = await deployAccessListContract( signer, networkArtifacts.AccessListFactory, diff --git a/src/test/integration/download.test.ts b/src/test/integration/download.test.ts index 3dc01799d..12a138285 100644 --- a/src/test/integration/download.test.ts +++ b/src/test/integration/download.test.ts @@ -90,7 +90,11 @@ describe('[Download Flow] - Should run a complete node flow.', () => { config = await getConfiguration(true) // Force reload the configuration database = await Database.init(config.dbConfig) oceanNode = await OceanNode.getInstance(config, database) - indexer = new OceanIndexer(database, config.indexingNetworks) + indexer = new OceanIndexer( + database, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) let network = getOceanArtifactsAdressesByChainId(DEVELOPMENT_CHAIN_ID) @@ -110,17 +114,18 @@ describe('[Download Flow] - Should run a complete node flow.', () => { }) it('should get node status', async () => { - const oceanNodeConfig = await getConfiguration(true) - const statusCommand = { command: PROTOCOL_COMMANDS.STATUS, - node: oceanNodeConfig.keys.peerId.toString() + node: oceanNode.getKeyManager().getPeerId().toString() } const response = await new StatusHandler(oceanNode).handle(statusCommand) assert(response.status.httpStatus === 200, 'http status not 200') const resp = await streamToString(response.stream as Readable) const status = JSON.parse(resp) - assert(status.id === oceanNodeConfig.keys.peerId.toString(), 'peer id not matching ') + assert( + status.id === oceanNode.getKeyManager().getPeerId().toString(), + 'peer id not matching ' + ) assert( status.allowedAdmins?.addresses[0]?.toLowerCase() === '0xe2DD09d719Da89e5a3D0F2549c7E24566e947260'?.toLowerCase(), @@ -131,11 +136,9 @@ describe('[Download Flow] - Should run a complete node flow.', () => { }) it('should get node detailed status', async () => { - const oceanNodeConfig = await getConfiguration(true) - const statusCommand = { command: PROTOCOL_COMMANDS.DETAILED_STATUS, - node: oceanNodeConfig.keys.peerId.toString() + node: oceanNode.getKeyManager().getPeerId().toString() } const response = await new DetailedStatusHandler(oceanNode).handle(statusCommand) assert(response.status.httpStatus === 200, 'http status not 200') diff --git a/src/test/integration/encryptDecryptDDO.test.ts b/src/test/integration/encryptDecryptDDO.test.ts index bd8422bf0..2a40b5942 100644 --- a/src/test/integration/encryptDecryptDDO.test.ts +++ b/src/test/integration/encryptDecryptDDO.test.ts @@ -19,7 +19,6 @@ import { import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC721Template.sol/ERC721Template.json' with { type: 'json' } import { genericDDO } from '../data/ddo.js' import { createHash } from 'crypto' -import { encrypt } from '../../utils/crypt.js' import { Database } from '../../components/database/index.js' import { DecryptDdoHandler } from '../../components/core/handler/ddoHandler.js' import { @@ -107,7 +106,11 @@ describe('Should encrypt and decrypt DDO', () => { database = await Database.init(config.dbConfig) oceanNode = OceanNode.getInstance(config, database) // will be used later - indexer = new OceanIndexer(database, mockSupportedNetworks) + indexer = new OceanIndexer( + database, + mockSupportedNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) }) @@ -157,7 +160,9 @@ describe('Should encrypt and decrypt DDO', () => { '0x' + createHash('sha256').update(JSON.stringify(genericAsset)).digest('hex') const genericAssetData = Uint8Array.from(Buffer.from(JSON.stringify(genericAsset))) - const encryptedData = await encrypt(genericAssetData, EncryptMethod.ECIES) + const encryptedData = await oceanNode + .getKeyManager() + .encrypt(genericAssetData, EncryptMethod.ECIES) encryptedMetaData = hexlify(encryptedData) const setMetaDataTx = await nftContract.setMetaData( @@ -213,10 +218,9 @@ describe('Should encrypt and decrypt DDO', () => { }) it('should authorize decrypter since is this node', async () => { - const config = await getConfiguration() const decryptDDOTask: DecryptDDOCommand = { command: PROTOCOL_COMMANDS.DECRYPT_DDO, - decrypterAddress: await config.keys.ethAddress, + decrypterAddress: oceanNode.getKeyManager().getEthAddress(), chainId, nonce: Date.now().toString(), signature: '0x123' diff --git a/src/test/integration/encryptFile.test.ts b/src/test/integration/encryptFile.test.ts index 1f8134338..988cf978a 100644 --- a/src/test/integration/encryptFile.test.ts +++ b/src/test/integration/encryptFile.test.ts @@ -54,7 +54,7 @@ describe('Encrypt File', () => { const expectedHeaders = { 'Content-Type': 'application/octet-stream', - 'X-Encrypted-By': config.keys.peerId.toString(), + 'X-Encrypted-By': oceanNode.getKeyManager().getPeerId().toString(), 'X-Encrypted-Method': EncryptMethod.AES } expect(response.status.headers).to.deep.equal(expectedHeaders) @@ -76,7 +76,7 @@ describe('Encrypt File', () => { expect(response.stream).to.be.instanceOf(Readable) const expectedHeaders = { 'Content-Type': 'application/octet-stream', - 'X-Encrypted-By': config.keys.peerId.toString(), + 'X-Encrypted-By': oceanNode.getKeyManager().getPeerId().toString(), 'X-Encrypted-Method': EncryptMethod.AES } expect(response.status.headers).to.deep.equal(expectedHeaders) @@ -98,7 +98,7 @@ describe('Encrypt File', () => { expect(response.stream).to.be.instanceOf(Readable) const expectedHeaders = { 'Content-Type': 'application/octet-stream', - 'X-Encrypted-By': config.keys.peerId.toString(), + 'X-Encrypted-By': oceanNode.getKeyManager().getPeerId().toString(), 'X-Encrypted-Method': EncryptMethod.ECIES } expect(response.status.headers).to.deep.equal(expectedHeaders) diff --git a/src/test/integration/indexer.test.ts b/src/test/integration/indexer.test.ts index f0a894276..ee7d32a3b 100644 --- a/src/test/integration/indexer.test.ts +++ b/src/test/integration/indexer.test.ts @@ -16,12 +16,9 @@ import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templat import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20TemplateEnterprise.sol/ERC20TemplateEnterprise.json' with { type: 'json' } import { Database } from '../../components/database/index.js' import { DatabaseFactory } from '../../components/database/DatabaseFactory.js' -import { - INDEXER_CRAWLING_EVENT_EMITTER, - OceanIndexer -} from '../../components/Indexer/index.js' +import { OceanIndexer } from '../../components/Indexer/index.js' import { RPCS } from '../../@types/blockchain.js' -import { getEventFromTx, sleep, streamToObject } from '../../utils/util.js' +import { getEventFromTx, streamToObject } from '../../utils/util.js' import { waitToIndex, expectedTimeoutFailure } from './testUtils.js' import { genericDDO } from '../data/ddo.js' import { @@ -34,7 +31,6 @@ import { Asset, DDO } from '@oceanprotocol/ddo-js' import { DEFAULT_TEST_TIMEOUT, OverrideEnvConfig, - TEST_ENV_CONFIG_FILE, buildEnvOverrideConfig, getMockSupportedNetworks, setupEnvironment, @@ -43,21 +39,13 @@ import { import { ENVIRONMENT_VARIABLES, EVENTS, - INDEXER_CRAWLING_EVENTS, PROTOCOL_COMMANDS } from '../../utils/constants.js' import { homedir } from 'os' import { QueryDdoStateHandler } from '../../components/core/handler/queryHandler.js' import { OceanNode } from '../../OceanNode.js' import { QueryCommand } from '../../@types/commands.js' -import { - getDeployedContractBlock, - getNetworkHeight -} from '../../components/Indexer/utils.js' -import { Blockchain } from '../../utils/blockchain.js' import { getConfiguration } from '../../utils/config.js' -import { OceanNodeConfig } from '../../@types/OceanNode.js' -import { encrypt } from '../../utils/crypt.js' import { EncryptMethod } from '../../@types/fileObject.js' import { deleteIndexedMetadataIfExists } from '../../utils/asset.js' @@ -113,7 +101,11 @@ describe('Indexer stores a new metadata events and orders.', () => { const config = await getConfiguration(true) database = await Database.init(config.dbConfig) oceanNode = OceanNode.getInstance(config, database) - indexer = new OceanIndexer(database, mockSupportedNetworks) + indexer = new OceanIndexer( + database, + mockSupportedNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) let artifactsAddresses = getOceanArtifactsAdressesByChainId(DEVELOPMENT_CHAIN_ID) if (!artifactsAddresses) { @@ -186,7 +178,9 @@ describe('Indexer stores a new metadata events and orders.', () => { const data = Uint8Array.from( Buffer.from(JSON.stringify(genericAsset.services[0].files)) ) - const encryptedData = await encrypt(data, EncryptMethod.ECIES) + const encryptedData = await oceanNode + .getKeyManager() + .encrypt(data, EncryptMethod.ECIES) const encryptedDataString = encryptedData.toString('hex') genericAsset.services[0].files = encryptedDataString const stringDDO = JSON.stringify(genericAsset) @@ -665,65 +659,3 @@ describe('Indexer stores a new metadata events and orders.', () => { indexer.stopAllThreads() }) }) - -describe('OceanIndexer - crawler threads', () => { - let envOverrides: OverrideEnvConfig[] - let config: OceanNodeConfig - let db: Database - let blockchain: Blockchain - - let oceanIndexer: OceanIndexer - const supportedNetworks: RPCS = getMockSupportedNetworks() - const chainID = DEVELOPMENT_CHAIN_ID.toString() - - let netHeight = 0 - let deployBlock = 0 - let startingBlock = 0 - - before(async () => { - config = await getConfiguration(true) - blockchain = new Blockchain( - supportedNetworks[chainID].rpc, - supportedNetworks[chainID].chainId, - config, - [] - ) - - deployBlock = getDeployedContractBlock(supportedNetworks[chainID].chainId) - netHeight = await getNetworkHeight(blockchain.getProvider()) - startingBlock = deployBlock + 1 - supportedNetworks[chainID].startBlock = startingBlock - - envOverrides = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.RPCS, ENVIRONMENT_VARIABLES.ADDRESS_FILE], - [ - JSON.stringify(supportedNetworks), - `${homedir}/.ocean/ocean-contracts/artifacts/address.json` - ] - ) - envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) - db = await Database.init(config.dbConfig) - }) - - it('should start a worker thread and handle RPCS "startBlock"', async () => { - INDEXER_CRAWLING_EVENT_EMITTER.addListener( - INDEXER_CRAWLING_EVENTS.CRAWLING_STARTED, - (data: any) => { - const { startBlock, contractDeploymentBlock, networkHeight } = data - expect(startBlock).to.be.equal(startingBlock) - expect(contractDeploymentBlock).to.be.equal(deployBlock) - expect(networkHeight).to.be.equal(netHeight) - } - ) - oceanIndexer = new OceanIndexer(db, supportedNetworks) - await sleep(DEFAULT_TEST_TIMEOUT / 2) - }) - - after(async () => { - await tearDownEnvironment(envOverrides) - oceanIndexer.stopAllThreads() - INDEXER_CRAWLING_EVENT_EMITTER.removeAllListeners( - INDEXER_CRAWLING_EVENTS.CRAWLING_STARTED - ) - }) -}) diff --git a/src/test/integration/operationsDashboard.test.ts b/src/test/integration/operationsDashboard.test.ts index ad6176afb..5f4cddcd6 100644 --- a/src/test/integration/operationsDashboard.test.ts +++ b/src/test/integration/operationsDashboard.test.ts @@ -53,6 +53,8 @@ import { ReindexTask } from '../../components/Indexer/ChainIndexer.js' import { create256Hash } from '../../utils/crypt.js' import { CollectFeesHandler } from '../../components/core/admin/collectFeesHandler.js' import { getProviderFeeToken } from '../../components/core/utils/feesHandler.js' +import { KeyManager } from '../../components/KeyManager/index.js' +import { BlockchainRegistry } from '../../components/BlockchainRegistry/index.js' describe('Should test admin operations', () => { let config: OceanNodeConfig @@ -106,8 +108,22 @@ describe('Should test admin operations', () => { config = await getConfiguration(true) // Force reload the configuration dbconn = await Database.init(config.dbConfig) - oceanNode = await OceanNode.getInstance(config, dbconn) - indexer = new OceanIndexer(dbconn, config.indexingNetworks) + const keyManager = new KeyManager(config) + const blockchainRegistry = new BlockchainRegistry(keyManager, config) + oceanNode = await OceanNode.getInstance( + config, + dbconn, + null, + null, + null, + keyManager, + blockchainRegistry + ) + indexer = new OceanIndexer( + dbconn, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) }) @@ -120,7 +136,7 @@ describe('Should test admin operations', () => { const stopNodeCommand: AdminStopNodeCommand = { command: PROTOCOL_COMMANDS.STOP_NODE, - node: config.keys.peerId.toString(), + node: oceanNode.getKeyManager().getPeerId().toString(), expiryTimestamp, signature } @@ -225,7 +241,7 @@ describe('Should test admin operations', () => { const reindexTxCommand: AdminReindexTxCommand = { command: PROTOCOL_COMMANDS.REINDEX_TX, - node: config.keys.peerId.toString(), + node: oceanNode.getKeyManager().getPeerId().toString(), txId: publishedDataset.trxReceipt.hash, chainId: DEVELOPMENT_CHAIN_ID, expiryTimestamp, @@ -300,7 +316,7 @@ describe('Should test admin operations', () => { } else { const reindexChainCommand: AdminReindexChainCommand = { command: PROTOCOL_COMMANDS.REINDEX_CHAIN, - node: config.keys.peerId.toString(), + node: oceanNode.getKeyManager().getPeerId().toString(), chainId: DEVELOPMENT_CHAIN_ID, expiryTimestamp, signature diff --git a/src/test/integration/pricing.test.ts b/src/test/integration/pricing.test.ts index 0c998f774..34517cabc 100644 --- a/src/test/integration/pricing.test.ts +++ b/src/test/integration/pricing.test.ts @@ -37,7 +37,6 @@ import { ENVIRONMENT_VARIABLES, EVENTS } from '../../utils/constants.js' import { homedir } from 'os' import { OceanNode } from '../../OceanNode.js' import { getConfiguration } from '../../utils/config.js' -import { encrypt } from '../../utils/crypt.js' import { EncryptMethod } from '../../@types/fileObject.js' import { Asset } from '@oceanprotocol/ddo-js' @@ -86,7 +85,11 @@ describe('Publish pricing scehmas and assert ddo stats - FRE & Dispenser', () => const config = await getConfiguration(true) database = await Database.init(config.dbConfig) oceanNode = await OceanNode.getInstance() - indexer = new OceanIndexer(database, mockSupportedNetworks) + indexer = new OceanIndexer( + database, + mockSupportedNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) artifactsAddresses = getOceanArtifactsAdressesByChainId(DEVELOPMENT_CHAIN_ID) if (!artifactsAddresses) { @@ -180,7 +183,9 @@ describe('Publish pricing scehmas and assert ddo stats - FRE & Dispenser', () => const data = Uint8Array.from( Buffer.from(JSON.stringify(genericAssetCloned.services[0].files)) ) - const encryptedData = await encrypt(data, EncryptMethod.ECIES) + const encryptedData = await oceanNode + .getKeyManager() + .encrypt(data, EncryptMethod.ECIES) const encryptedDataString = encryptedData.toString('hex') genericAssetCloned.services[0].files = encryptedDataString const stringDDO = JSON.stringify(genericAssetCloned) @@ -316,7 +321,9 @@ describe('Publish pricing scehmas and assert ddo stats - FRE & Dispenser', () => const data = Uint8Array.from( Buffer.from(JSON.stringify(genericAssetCloned.services[1].files)) ) - const encryptedData = await encrypt(data, EncryptMethod.ECIES) + const encryptedData = await oceanNode + .getKeyManager() + .encrypt(data, EncryptMethod.ECIES) const encryptedDataString = encryptedData.toString('hex') genericAssetCloned.services[1].files = encryptedDataString const stringDDO = JSON.stringify(genericAssetCloned) diff --git a/src/test/integration/transactionValidation.test.ts b/src/test/integration/transactionValidation.test.ts index 007633327..974438716 100644 --- a/src/test/integration/transactionValidation.test.ts +++ b/src/test/integration/transactionValidation.test.ts @@ -1,5 +1,5 @@ import { expect, assert } from 'chai' -import { JsonRpcProvider, Signer } from 'ethers' +import { JsonRpcProvider, Signer, FallbackProvider } from 'ethers' import { validateOrderTransaction } from '../../components/core/utils/validateOrders.js' import { expectedTimeoutFailure, waitToIndex } from './testUtils.js' import { genericDDO } from '../data/ddo.js' @@ -29,6 +29,7 @@ describe('validateOrderTransaction Function with Orders', () => { let database: Database let oceanNode: OceanNode let provider: JsonRpcProvider + let fallbackProvider: FallbackProvider let publisherAccount: Signer let consumerAccount: Signer let consumerAddress: string @@ -69,7 +70,11 @@ describe('validateOrderTransaction Function with Orders', () => { config = await getConfiguration(true) // Force reload the configuration const dbconn = await Database.init(config.dbConfig) oceanNode = await OceanNode.getInstance(config, dbconn) - indexer = new OceanIndexer(dbconn, config.indexingNetworks) + indexer = new OceanIndexer( + dbconn, + config.indexingNetworks, + oceanNode.blockchainRegistry + ) oceanNode.addIndexer(indexer) let network = getOceanArtifactsAdressesByChainId(DEVELOPMENT_CHAIN_ID) @@ -78,6 +83,7 @@ describe('validateOrderTransaction Function with Orders', () => { } provider = new JsonRpcProvider('http://127.0.0.1:8545') + fallbackProvider = new FallbackProvider([provider]) publisherAccount = (await provider.getSigner(0)) as Signer consumerAccount = (await provider.getSigner(1)) as Signer @@ -149,7 +155,7 @@ describe('validateOrderTransaction Function with Orders', () => { const validationResult = await validateOrderTransaction( orderTxId, consumerAddress, - provider, + fallbackProvider, dataNftAddress, datatokenAddress, parseInt(serviceId), @@ -179,7 +185,7 @@ describe('validateOrderTransaction Function with Orders', () => { const validationResult = await validateOrderTransaction( reOrderTxId, consumerAddress, - provider, + fallbackProvider, dataNftAddress, datatokenAddress, parseInt(serviceId), @@ -198,7 +204,7 @@ describe('validateOrderTransaction Function with Orders', () => { const validationResult = await validateOrderTransaction( reOrderTxId, consumerAddress, - provider, + fallbackProvider, dataNftAddress, datatokenAddress, parseInt('999'), @@ -217,7 +223,7 @@ describe('validateOrderTransaction Function with Orders', () => { const validationResult = await validateOrderTransaction( reOrderTxId, '0x0', - provider, + fallbackProvider, dataNftAddress, datatokenAddress, parseInt(serviceId), diff --git a/src/test/unit/blockchain.test.ts b/src/test/unit/blockchain.test.ts index d22bdae0f..737157c19 100644 --- a/src/test/unit/blockchain.test.ts +++ b/src/test/unit/blockchain.test.ts @@ -1,6 +1,8 @@ import { expect } from 'chai' import { OceanNodeConfig } from '../../@types/OceanNode.js' -import { RPCS, SupportedNetwork } from '../../@types/blockchain.js' + +import { BlockchainRegistry } from '../../components/BlockchainRegistry/index.js' +import { KeyManager } from '../../components/KeyManager/index.js' import { Blockchain } from '../../utils/blockchain.js' import { getConfiguration } from '../../utils/config.js' import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' @@ -12,14 +14,12 @@ import { setupEnvironment, tearDownEnvironment } from '../utils/utils.js' -import { expectedTimeoutFailure } from '../integration/testUtils.js' + import { DEVELOPMENT_CHAIN_ID, KNOWN_CONFIDENTIAL_EVMS } from '../../utils/address.js' import { isConfidentialEVM } from '../../utils/asset.js' let envOverrides: OverrideEnvConfig[] let config: OceanNodeConfig -let rpcs: RPCS -let network: SupportedNetwork let blockchain: Blockchain describe('Should validate blockchain network connections', () => { before(async () => { @@ -31,46 +31,20 @@ describe('Should validate blockchain network connections', () => { ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) config = await getConfiguration(true) - - rpcs = config.supportedNetworks - network = rpcs['8996'] - blockchain = new Blockchain( - network.rpc, - network.chainId, - config, - network.fallbackRPCs - ) + const keyManager = new KeyManager(config) + const blockchainRegistry = new BlockchainRegistry(keyManager, config) + // network = rpcs['8996'] + blockchain = blockchainRegistry.getBlockchain(8996) }) it('should get known rpcs', () => { expect(blockchain.getKnownRPCs().length).to.be.equal(3) }) - it('should get network not ready (wrong RPC setting)', async () => { - const status = await blockchain.isNetworkReady() - expect(status.ready).to.be.equal(false) - }) - - it('should get network ready after retry other RPCs', async function () { + it('should get network not ready', async function () { this.timeout(DEFAULT_TEST_TIMEOUT * 2) - let status = await blockchain.isNetworkReady() + const status = await blockchain.isNetworkReady() expect(status.ready).to.be.equal(false) - // at least one should be OK - const retryResult = await blockchain.tryFallbackRPCs() - // ignore node network errors (on ci there are network issues sometimes) - // we can't do much if we have timeouts, bad urls or connections refused - if (!retryResult.ready && retryResult.error) { - const networkIssue = - retryResult.error.includes('TIMEOUT') || - retryResult.error.includes('ECONNREFUSED') || - retryResult.error.includes('ENOTFOUND') - - expect(expectedTimeoutFailure(this.test.title)).to.be.equal(networkIssue) - } else { - expect(retryResult.ready).to.be.equal(true) - status = await blockchain.isNetworkReady() - expect(status.ready).to.be.equal(true) - } }) it('should check if chain is confidential EVM', () => { diff --git a/src/test/unit/commands.test.ts b/src/test/unit/commands.test.ts index 63bd57852..ec4385fd6 100644 --- a/src/test/unit/commands.test.ts +++ b/src/test/unit/commands.test.ts @@ -2,7 +2,9 @@ import { expect } from 'chai' import { PROTOCOL_COMMANDS, SUPPORTED_PROTOCOL_COMMANDS } from '../../utils/index.js' import { CoreHandlersRegistry } from '../../components/core/handler/coreHandlersRegistry.js' import { BaseHandler } from '../../components/core/handler/handler.js' +import { getConfiguration } from '../../utils/config.js' import { OceanNode } from '../../OceanNode.js' +import { KeyManager } from '../../components/KeyManager/index.js' import { ComputeGetEnvironmentsCommand, ComputeGetResultCommand, @@ -52,9 +54,17 @@ import { CollectFeesHandler } from '../../components/core/admin/collectFeesHandl import { GetJobsHandler } from '../../components/core/handler/getJobs.js' describe('Commands and handlers', () => { + let node: OceanNode + before(async () => { + const config = await getConfiguration() + const keyManager = new KeyManager(config) + + node = OceanNode.getInstance(config, null, null, null, null, keyManager, null, true) + }) + it('Check that all supported commands have registered handlers', () => { // To make sure we do not forget to register handlers - const node: OceanNode = OceanNode.getInstance() + for (const command of SUPPORTED_PROTOCOL_COMMANDS) { expect(CoreHandlersRegistry.getInstance(node).getHandler(command)).to.be.instanceof( BaseHandler @@ -64,14 +74,12 @@ describe('Commands and handlers', () => { it('Check that supported commands and handlers match', () => { // To make sure we do not forget to register anything on supported commands - const node: OceanNode = OceanNode.getInstance() const handlers: string[] = node.getCoreHandlers().getRegisteredCommands() expect(SUPPORTED_PROTOCOL_COMMANDS.length).to.be.equal(handlers.length) }) it('Check that all commands are validating required parameters', () => { // To make sure we do not forget to register anything on supported commands - const node: OceanNode = OceanNode.getInstance() // downloadHandler const downloadHandler: DownloadHandler = CoreHandlersRegistry.getInstance( diff --git a/src/test/unit/credentials.test.ts b/src/test/unit/credentials.test.ts index 6cfb0f088..3fa2e7664 100644 --- a/src/test/unit/credentials.test.ts +++ b/src/test/unit/credentials.test.ts @@ -7,6 +7,8 @@ import { tearDownEnvironment, TEST_ENV_CONFIG_FILE } from '../utils/utils.js' +import { KeyManager } from '../../components/KeyManager/index.js' +import { BlockchainRegistry } from '../../components/BlockchainRegistry/index.js' import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' import { homedir } from 'os' import { Credentials } from '@oceanprotocol/ddo-js' @@ -14,6 +16,7 @@ import { CREDENTIALS_TYPES } from '../../@types/DDO/Credentials.js' import { Blockchain } from '../../utils/blockchain.js' import { Signer } from 'ethers' import { getConfiguration } from '../../utils/index.js' +import { DEVELOPMENT_CHAIN_ID } from '../../utils/address.js' let envOverrides: OverrideEnvConfig[] let blockchain: Blockchain @@ -24,15 +27,19 @@ describe('credentials', () => { envOverrides = buildEnvOverrideConfig( [ENVIRONMENT_VARIABLES.RPCS, ENVIRONMENT_VARIABLES.ADDRESS_FILE], [ - '{ "8996":{ "rpc":"http://172.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100 }}', + '{ "8996":{ "rpc":"http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100 }}', `${homedir}/.ocean/ocean-contracts/artifacts/address.json` ] ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) const config = await getConfiguration() // Initialize blockchain for tests - blockchain = new Blockchain('http://172.0.0.1:8545', 8996, config, []) - signer = blockchain.getSigner() + const keyManager = new KeyManager(config) + const blockchains = new BlockchainRegistry(keyManager, config) + blockchain = blockchains.getBlockchain(DEVELOPMENT_CHAIN_ID) + // force usage of known bad provider + await blockchain.getProvider(true) + signer = await blockchain.getSigner() }) it('should allow access with undefined or empty credentials', async () => { diff --git a/src/test/unit/crypt.test.ts b/src/test/unit/crypt.test.ts index 5624ee001..5fb349ab8 100644 --- a/src/test/unit/crypt.test.ts +++ b/src/test/unit/crypt.test.ts @@ -1,5 +1,4 @@ import { expect } from 'chai' -import { decrypt, encrypt } from '../../utils/crypt.js' import { EncryptMethod } from '../../@types/fileObject.js' import { buildEnvOverrideConfig, @@ -8,31 +7,36 @@ import { tearDownEnvironment, TEST_ENV_CONFIG_FILE } from '../utils/utils.js' +import { getConfiguration } from '../../utils/index.js' import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' import { homedir } from 'os' +import { KeyManager } from '../../components/KeyManager/index.js' describe('crypt', () => { let envOverrides: OverrideEnvConfig[] + let keyManager: KeyManager before(async () => { envOverrides = buildEnvOverrideConfig( [ENVIRONMENT_VARIABLES.RPCS, ENVIRONMENT_VARIABLES.ADDRESS_FILE], [ - '{ "8996":{ "rpc":"http://172.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100 }}', + '{ "8996":{ "rpc":"http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100 }}', `${homedir}/.ocean/ocean-contracts/artifacts/address.json` ] ) envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) + const config = await getConfiguration() + keyManager = new KeyManager(config) }) it('should encrypt/decrypt AES', async () => { const data = Uint8Array.from(Buffer.from('ocean')) - const encryptedData = await encrypt(data, EncryptMethod.AES) - const decryptedData = await decrypt(encryptedData, EncryptMethod.AES) + const encryptedData = await keyManager.encrypt(data, EncryptMethod.AES) + const decryptedData = await keyManager.decrypt(encryptedData, EncryptMethod.AES) expect(Uint8Array.from(decryptedData)).to.deep.equal(data) }) it('should encrypt/decrypt ECIES', async () => { const data = Uint8Array.from(Buffer.from('ocean')) - const encryptedData = await encrypt(data, EncryptMethod.ECIES) - const decryptedData = await decrypt(encryptedData, EncryptMethod.ECIES) + const encryptedData = await keyManager.encrypt(data, EncryptMethod.ECIES) + const decryptedData = await keyManager.decrypt(encryptedData, EncryptMethod.ECIES) expect(Uint8Array.from(decryptedData)).to.deep.equal(data) }) after(async () => { diff --git a/src/test/unit/download.test.ts b/src/test/unit/download.test.ts index 5a57d4a3f..0cfd291da 100644 --- a/src/test/unit/download.test.ts +++ b/src/test/unit/download.test.ts @@ -15,7 +15,6 @@ import { setupEnvironment, tearDownEnvironment } from '../utils/utils.js' -import { decrypt } from '../../utils/crypt.js' import { validateFilesStructure } from '../../components/core/handler/downloadHandler.js' import { AssetUtils, isConfidentialChainDDO } from '../../utils/asset.js' import { DEVELOPMENT_CHAIN_ID, KNOWN_CONFIDENTIAL_EVMS } from '../../utils/address.js' @@ -96,7 +95,9 @@ describe('Should validate files structure for download', () => { const data = Uint8Array.from(Buffer.from(serviceData.files.slice(2), 'hex')) - const decryptedUrlBytes = await decrypt(data, EncryptMethod.ECIES) + const decryptedUrlBytes = await oceanNode + .getKeyManager() + .decrypt(data, EncryptMethod.ECIES) // Convert the decrypted bytes back to a string const decryptedFilesString = Buffer.from(decryptedUrlBytes).toString() // back to JSON representation @@ -130,8 +131,19 @@ describe('Should validate files structure for download', () => { ).to.be.equal(false) // this encrypted file data if for assetURL with otherNFTAddress and otherDatatokenAddress above - const encryptedFilesData = - '0x04a8e18eb64ec339ec4438b7dea0155630928c2bbcec707589adb42b9359e8baf97a9eab822e47195599105909573d294de0f68125a38a47ca583e8171d5e636f6f710687363e65e62932457a8d0e4825dbbb7fb047a4b6013655d2925ae3756015040348d327bca02d12cede80d6d024cc7a690d4976635225a27fe6d8ca85354e72c5034c62962f3cdfe186460a84205bd14b71041d8b915bc9f94eae89a07210c12198afca09e3ba9d7c7df394c00a090b9e3631609de434b8f45adce16fa7f86dd0ac71168eefd819b6fb226a6a7371d3fab88def9ee1454b96e22fa0e9408bdc0ec38cd6bd067476cc0a33af3fb8ed4e4675b45e452bb687f485ce94f8e4e72ebb816c76fae5c9408e66fd89d0251ac6b9a34deaac8dbd22439a90ea9cc4bf80028e3e93259d468820ddf18f83e68781422d20bc19a2a3c2a0d93426cff6a8d13270e75732e1dd9ec8a8d4621b9f9c9c136e9ae48336da1844b1826' + const newAssetURL = structuredClone(assetURL) + newAssetURL.nftAddress = otherNFTAddress + newAssetURL.datatokenAddress = otherDatatokenAddress + const result = await new EncryptHandler(oceanNode).handle({ + blob: JSON.stringify(newAssetURL), + encoding: 'string', + encryptionType: EncryptMethod.ECIES, + command: PROTOCOL_COMMANDS.ENCRYPT + }) + + const encryptedFilesData: string = await streamToString(result.stream as Readable) + // const encryptedFilesData = + /// '0x04a8e18eb64ec339ec4438b7dea0155630928c2bbcec707589adb42b9359e8baf97a9eab822e47195599105909573d294de0f68125a38a47ca583e8171d5e636f6f710687363e65e62932457a8d0e4825dbbb7fb047a4b6013655d2925ae3756015040348d327bca02d12cede80d6d024cc7a690d4976635225a27fe6d8ca85354e72c5034c62962f3cdfe186460a84205bd14b71041d8b915bc9f94eae89a07210c12198afca09e3ba9d7c7df394c00a090b9e3631609de434b8f45adce16fa7f86dd0ac71168eefd819b6fb226a6a7371d3fab88def9ee1454b96e22fa0e9408bdc0ec38cd6bd067476cc0a33af3fb8ed4e4675b45e452bb687f485ce94f8e4e72ebb816c76fae5c9408e66fd89d0251ac6b9a34deaac8dbd22439a90ea9cc4bf80028e3e93259d468820ddf18f83e68781422d20bc19a2a3c2a0d93426cff6a8d13270e75732e1dd9ec8a8d4621b9f9c9c136e9ae48336da1844b1826' const sameDDOOtherFiles = ddoObj sameDDOOtherFiles.services[0].files = encryptedFilesData expect( @@ -140,7 +152,9 @@ describe('Should validate files structure for download', () => { const data = Uint8Array.from(Buffer.from(encryptedFilesData.slice(2), 'hex')) - const decryptedUrlBytes = await decrypt(data, EncryptMethod.ECIES) + const decryptedUrlBytes = await oceanNode + .getKeyManager() + .decrypt(data, EncryptMethod.ECIES) // Convert the decrypted bytes back to a string const decryptedFilesString = Buffer.from(decryptedUrlBytes).toString() // back to JSON representation diff --git a/src/test/unit/indexer/validation.test.ts b/src/test/unit/indexer/validation.test.ts index 205614b18..956668b4e 100644 --- a/src/test/unit/indexer/validation.test.ts +++ b/src/test/unit/indexer/validation.test.ts @@ -70,6 +70,8 @@ describe('Schema validation tests', () => { undefined, undefined, undefined, + undefined, + undefined, true ) }) diff --git a/src/test/unit/networking.test.ts b/src/test/unit/networking.test.ts index d9c98da95..8a88b2bda 100644 --- a/src/test/unit/networking.test.ts +++ b/src/test/unit/networking.test.ts @@ -17,7 +17,8 @@ import { } from '../utils/utils.js' import { OceanNode } from '../../OceanNode.js' import { StatusHandler } from '../../components/core/handler/statusHandler.js' -import { CoreHandlersRegistry } from '../../components/core/handler/coreHandlersRegistry.js' +import { KeyManager } from '../../components/KeyManager/index.js' +import { OceanP2P } from '../../components/P2P/index.js' let envOverrides: OverrideEnvConfig[] @@ -129,8 +130,7 @@ describe('Test rate limitations and deny list defaults', () => { }) describe('Test rate limitations and deny list settings', () => { - const node: OceanNode = OceanNode.getInstance() - + let node: OceanNode before(async () => { envOverrides = buildEnvOverrideConfig( [ @@ -147,7 +147,12 @@ describe('Test rate limitations and deny list settings', () => { 3 ] ) - await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) + envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) + const config = await getConfiguration(true) + const keyManager = new KeyManager(config) + const p2pNode = new OceanP2P(config, keyManager) + await p2pNode.start() + node = OceanNode.getInstance(config, null, p2pNode, null, null, null, null, true) }) it('should read deny list of other peers and ips', async () => { @@ -164,17 +169,17 @@ describe('Test rate limitations and deny list settings', () => { it('Test rate limit per IP, on handler', async () => { // need to set it here, on a running node is done at request/middleware level - node.setRemoteCaller('127.0.0.1') - const statusHandler: StatusHandler = CoreHandlersRegistry.getInstance( - node - ).getHandler(PROTOCOL_COMMANDS.STATUS) + const statusHandler: StatusHandler = node + .getP2PNode() + .getCoreHandlers() + .getHandler(PROTOCOL_COMMANDS.STATUS) - const rate = await statusHandler.checkRateLimit() + const rate = await statusHandler.checkRateLimit('127.0.0.2') const rateLimitResponses = [] expect(rate).to.be.equal(true) for (let i = 0; i < 4; i++) { // 4 responses, at least one should be blocked - const rateResp = await statusHandler.checkRateLimit() + const rateResp = await statusHandler.checkRateLimit('127.0.0.2') rateLimitResponses.push(rateResp) } const filtered = rateLimitResponses.filter((r) => r === false) @@ -184,16 +189,15 @@ describe('Test rate limitations and deny list settings', () => { it('Test rate limit per IP, on handler, different IPs', async () => { // need to set it here, on a running node is done at request/middleware level // none will be blocked, since its always another caller - const ips = ['127.0.0.2', '127.0.0.3', '127.0.0.4', '127.0.0.5'] - + const ips = ['127.0.0.3', '127.0.0.4', '127.0.0.5', '127.0.0.6'] const rateLimitResponses = [] - const statusHandler: StatusHandler = CoreHandlersRegistry.getInstance( - node - ).getHandler(PROTOCOL_COMMANDS.STATUS) + const statusHandler: StatusHandler = node + .getP2PNode() + .getCoreHandlers() + .getHandler(PROTOCOL_COMMANDS.STATUS) for (let i = 0; i < ips.length; i++) { - node.setRemoteCaller(ips[i]) - const rateResp = await statusHandler.checkRateLimit() + const rateResp = await statusHandler.checkRateLimit(ips[i]) rateLimitResponses.push(rateResp) } const filtered = rateLimitResponses.filter((r) => r === true) @@ -208,15 +212,15 @@ describe('Test rate limitations and deny list settings', () => { const rateLimitResponses = [] - const statusHandler: StatusHandler = CoreHandlersRegistry.getInstance( - node - ).getHandler(PROTOCOL_COMMANDS.STATUS) + const statusHandler: StatusHandler = node + .getP2PNode() + .getCoreHandlers() + .getHandler(PROTOCOL_COMMANDS.STATUS) const aboveLimit = 20 for (let i = 0, x = 0; i < CONNECTION_HISTORY_DELETE_THRESHOLD + aboveLimit; i++) { const ip = originalIPPiece + x // start at 127.0.0.2 - node.setRemoteCaller(ip) - const rateResp = await statusHandler.checkRateLimit() + const rateResp = await statusHandler.checkRateLimit(ip) rateLimitResponses.push(rateResp) x++ // start back diff --git a/src/test/unit/ocean.test.ts b/src/test/unit/ocean.test.ts index 67b2d8886..d2969b164 100644 --- a/src/test/unit/ocean.test.ts +++ b/src/test/unit/ocean.test.ts @@ -1,6 +1,8 @@ import { OceanNode } from '../../OceanNode.js' import { OceanIndexer } from '../../components/Indexer/index.js' import { OceanP2P } from '../../components/P2P/index.js' +import { KeyManager } from '../../components/KeyManager/index.js' +import { BlockchainRegistry } from '../../components/BlockchainRegistry/index.js' import { OceanProvider } from '../../components/Provider/index.js' import { Database } from '../../components/database/index.js' import { ENVIRONMENT_VARIABLES, getConfiguration } from '../../utils/index.js' @@ -38,8 +40,10 @@ describe('Status command tests', async () => { // because of this const config = await getConfiguration(true) const db = await Database.init(config.dbConfig) - const oceanP2P = new OceanP2P(config, db) - const oceanIndexer = new OceanIndexer(db, config.indexingNetworks) + const keyManager = new KeyManager(config) + const blockchainRegistry = new BlockchainRegistry(keyManager, config) + const oceanP2P = new OceanP2P(config, keyManager, db) + const oceanIndexer = new OceanIndexer(db, config.indexingNetworks, blockchainRegistry) const oceanProvider = new OceanProvider(db) const oceanNode = OceanNode.getInstance(config, db, oceanP2P) diff --git a/src/test/unit/oceanP2P.test.ts b/src/test/unit/oceanP2P.test.ts index 020448230..3c0cf82aa 100644 --- a/src/test/unit/oceanP2P.test.ts +++ b/src/test/unit/oceanP2P.test.ts @@ -1,6 +1,7 @@ import { assert } from 'chai' import { getConfiguration } from '../../utils/config.js' import { OceanP2P } from '../../components/P2P/index.js' +import { KeyManager } from '../../components/KeyManager/index.js' import { delay } from '../integration/testUtils.js' import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' import { @@ -16,6 +17,8 @@ describe('OceanP2P Test', () => { let node2: OceanP2P let config1: OceanNodeConfig let config2: OceanNodeConfig + let peerId1: string + let peerId2: string const mDNSInterval: number = 1 const envOverrides = buildEnvOverrideConfig( @@ -38,14 +41,19 @@ describe('OceanP2P Test', () => { process.env.PRIVATE_KEY = process.env.NODE1_PRIVATE_KEY config1 = await getConfiguration(true) config1.p2pConfig.ipV4BindTcpPort = 0 + config1.p2pConfig.ipV6BindTcpPort = 0 + config1.p2pConfig.ipV4BindWsPort = 0 config1.p2pConfig.ipV4BindWssPort = 0 + config1.p2pConfig.ipV6BindWsPort = 0 // we don't need bootstrap nodes, we rely on Multicast DNS config1.p2pConfig.mDNSInterval = mDNSInterval * 1e3 config1.p2pConfig.bootstrapNodes = [] // enable private IP config1.p2pConfig.announcePrivateIp = true config1.p2pConfig.filterAnnouncedAddresses = ['172.15.0.0/24'] - node1 = new OceanP2P(config1, null) + const keyManager = new KeyManager(config1) + node1 = new OceanP2P(config1, keyManager) + peerId1 = keyManager.getPeerIdString() await node1.start() assert(node1, 'Failed to create P2P Node instance') }) @@ -63,46 +71,30 @@ describe('OceanP2P Test', () => { // enable private IP config2.p2pConfig.announcePrivateIp = true config2.p2pConfig.filterAnnouncedAddresses = ['172.15.0.0/24'] // allow nodes to see each other locally for tests - node2 = new OceanP2P(config2, null) + const keyManager2 = new KeyManager(config2) + node2 = new OceanP2P(config2, keyManager2) + peerId2 = keyManager2.getPeerIdString() await node2.start() assert(node2, 'Failed to create P2P Node instance') }) it('Start check peerID of each node', () => { - assert( - config1.keys.peerId.toString() === node1._libp2p.peerId.toString(), - 'Peer missmatch for node1' - ) - assert( - config2.keys.peerId.toString() === node2._libp2p.peerId.toString(), - 'Peer missmatch for node2' - ) + assert(peerId1 === node1._libp2p.peerId.toString(), 'Peer missmatch for node1') + assert(peerId2 === node2._libp2p.peerId.toString(), 'Peer missmatch for node2') }) delay(mDNSInterval * 1e3 * 2) it('Start check if nodes are connected', async () => { const allPeers1 = await node1.getAllPeerStore() const peers1 = allPeers1.map((a: any) => a.id.toString()) - assert( - peers1.includes(config2.keys.peerId.toString()), - 'Node2 not found in node1 peer list' - ) + assert(peers1.includes(peerId2), 'Node2 not found in node1 peer list') const allPeers2 = await node2.getAllPeerStore() const peers2 = allPeers2.map((a: any) => a.id.toString()) - assert( - peers2.includes(config1.keys.peerId.toString()), - 'Node1 not found in node2 peer list' - ) + assert(peers2.includes(peerId1), 'Node1 not found in node2 peer list') }) it('Start check if nodes are connected with pubsub', async () => { let peers = await node1.getOceanPeers() - assert( - peers.includes(config2.keys.peerId.toString()), - 'Node2 not found in node1 peer list' - ) + assert(peers.includes(peerId2), 'Node2 not found in node1 peer list') peers = await node2.getOceanPeers() - assert( - peers.includes(config1.keys.peerId.toString()), - 'Node1 not found in node2 peer list' - ) + assert(peers.includes(peerId1), 'Node1 not found in node2 peer list') }) after(() => { @@ -117,7 +109,8 @@ describe('OceanP2P Test without DB_URL set', () => { config.dbConfig.url = '' config.dbConfig.dbType = null // assert(config.dbConfig.url === '', 'DB URL should not be set') - const p2pNode = new OceanP2P(config) + const keyManager = new KeyManager(config) + const p2pNode = new OceanP2P(config, keyManager) assert(p2pNode, 'Failed to create P2P Node instance') assert(config.p2pConfig, 'Failed to get P2P Node config') // This is not testing P2P Node at all, just checking configuration diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts index 5ea056e39..5a2f200f8 100644 --- a/src/test/unit/storage.test.ts +++ b/src/test/unit/storage.test.ts @@ -19,12 +19,11 @@ import { } from '../utils/utils.js' import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' import { getConfiguration } from '../../utils/index.js' -import { Readable } from 'stream' -import fs from 'fs' import { expectedTimeoutFailure } from '../integration/testUtils.js' -let nodeId: string - +// let nodeId: string +const nodeId = '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72' +const nodeId2 = '16Uiu2HAmQWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq73' describe('URL Storage tests', () => { let file: any = { type: 'url', @@ -36,7 +35,7 @@ describe('URL Storage tests', () => { Authorization: 'Bearer auth_token_X' } ], - encryptedBy: '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72', + encryptedBy: nodeId, encryptMethod: EncryptMethod.AES } let storage: Storage @@ -58,11 +57,7 @@ describe('URL Storage tests', () => { }) it('canDecrypt should return true for the correct nodeId', () => { - assert( - storage.canDecrypt('16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72') === - true, - "can't decrypt with the correct nodeId" - ) + assert(storage.canDecrypt(nodeId) === true, "can't decrypt with the correct nodeId") }) it('canDecrypt should return false for an incorrect nodeId', () => { @@ -584,37 +579,10 @@ describe('URL Storage encryption tests', () => { it('canDecrypt should return false when the file is not encrypted', () => { assert( - storage.canDecrypt('16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72') === - false, + storage.canDecrypt(nodeId) === false, 'Wrong response from canDecrypt() for an unencrypted file' ) }) - - it('encrypt method should correctly encrypt data', async () => { - const { keys } = config - nodeId = keys.peerId.toString() - // Perform encryption - const encryptResponse = await storage.encrypt(EncryptMethod.AES) - assert(encryptResponse.httpStatus === 200, 'Response is not 200') - assert(encryptResponse.stream, 'Stream is not null') - assert(encryptResponse.stream instanceof Readable, 'Stream is not a ReadableStream') - - // Create a writable stream for the output file - const fileStream = fs.createWriteStream('src/test/data/organizations-100.aes') - - // Use the 'finish' event to know when the file has been fully written - fileStream.on('finish', () => { - console.log('Encrypted file has been written successfully') - }) - - // Handle errors in the stream - encryptResponse.stream.on('error', (err) => { - console.error('Stream encountered an error:', err) - }) - - // Pipe the encrypted content stream to the file stream - encryptResponse.stream.pipe(fileStream) - }) }) describe('URL Storage encryption tests', function () { @@ -677,8 +645,11 @@ describe('URL Storage encryption tests', function () { it('canDecrypt should return false when called from an unauthorised node', () => { assert( - storage.canDecrypt('16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72') === - false, + storage.canDecrypt(nodeId) === true, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + assert( + storage.canDecrypt(nodeId2) === false, 'Wrong response from canDecrypt() for an unencrypted file' ) }) diff --git a/src/test/utils/assets.ts b/src/test/utils/assets.ts index 959c040f7..75d9a888d 100644 --- a/src/test/utils/assets.ts +++ b/src/test/utils/assets.ts @@ -20,20 +20,16 @@ import { import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20TemplateEnterprise.sol/ERC20TemplateEnterprise.json' with { type: 'json' } import { getEventFromTx, streamToObject } from '../../utils/util.js' -import { encrypt } from '../../utils/crypt.js' import { AssetUtils } from '../../utils/asset.js' -import { - DDO_IDENTIFIER_PREFIX, - PROTOCOL_COMMANDS, - getConfiguration -} from '../../utils/index.js' +import { DDO_IDENTIFIER_PREFIX, PROTOCOL_COMMANDS } from '../../utils/index.js' import { FeesHandler } from '../../components/core/handler/feesHandler.js' import { OceanNode } from '../../OceanNode.js' import { ProviderFees } from '../../@types/Fees.js' export async function publishAsset(asset: any, publisherAccount: Signer) { const genericAsset = JSON.parse(JSON.stringify(asset)) + const oceanNode = OceanNode.getInstance() try { let network = getOceanArtifactsAdressesByChainId(DEVELOPMENT_CHAIN_ID) if (!network) { @@ -85,7 +81,9 @@ export async function publishAsset(asset: any, publisherAccount: Signer) { const data = Uint8Array.from( Buffer.from(JSON.stringify(genericAsset.services[0].files)) ) - const encryptedData = await encrypt(data, EncryptMethod.ECIES) + const encryptedData = await oceanNode + .getKeyManager() + .encrypt(data, EncryptMethod.ECIES) const encryptedDataString = encryptedData.toString('hex') const nftContract = new ethers.Contract( @@ -153,13 +151,12 @@ export async function orderAsset( ) if (!providerFees) { - const oceanNodeConfig = await getConfiguration(true) const getFeesCommand = { command: PROTOCOL_COMMANDS.GET_FEES, ddoId: genericAsset.id, serviceId: service.id, consumerAddress, - node: oceanNodeConfig.keys.peerId.toString() + node: oceanNode.getKeyManager().getPeerId().toString() } const response = await new FeesHandler(oceanNode).handle(getFeesCommand) const fees = await streamToObject(response.stream as Readable) @@ -239,13 +236,12 @@ export async function reOrderAsset( ) if (!providerFees) { - const oceanNodeConfig = await getConfiguration(true) const statusCommand = { command: PROTOCOL_COMMANDS.GET_FEES, ddoId: genericAsset.id, serviceId: service.id, consumerAddress, - node: oceanNodeConfig.keys.peerId.toString() + node: oceanNode.getKeyManager().getPeerId().toString() } const response = await new FeesHandler(oceanNode).handle(statusCommand) const fees = await streamToObject(response.stream as Readable) diff --git a/src/utils/blockchain.ts b/src/utils/blockchain.ts index ff06f7d89..11edf4473 100644 --- a/src/utils/blockchain.ts +++ b/src/utils/blockchain.ts @@ -5,20 +5,19 @@ import { Contract, JsonRpcApiProvider, JsonRpcProvider, + FallbackProvider, isAddress, - Network, parseUnits, Wallet, - TransactionReceipt, - sha512 + TransactionReceipt } from 'ethers' import { getConfiguration } from './config.js' import { CORE_LOGGER } from './logging/common.js' -import { sleep } from './util.js' import { ConnectionStatus } from '../@types/blockchain.js' import { ValidateChainId } from '../@types/commands.js' import { KNOWN_CONFIDENTIAL_EVMS } from '../utils/address.js' import { OceanNodeConfig } from '../@types/OceanNode.js' +import { KeyManager } from '../components/KeyManager/index.js' const MIN_GAS_FEE_POLYGON = 30000000000 // minimum recommended 30 gwei polygon main and mumbai fees const MIN_GAS_FEE_SEPOLIA = 4000000000 // minimum 4 gwei for eth sepolia testnet @@ -28,80 +27,84 @@ const MUMBAI_NETWORK_ID = 80001 const SEPOLIA_NETWORK_ID = 11155111 export class Blockchain { - private config: OceanNodeConfig + private config?: OceanNodeConfig // Optional for new constructor private static signers: Map = new Map() private static providers: Map = new Map() + private keyManager: KeyManager private signer: Signer - private provider: JsonRpcApiProvider + private provider: FallbackProvider + private providers: JsonRpcProvider[] = [] private chainId: number private knownRPCs: string[] = [] - private networkAvailable: boolean = false + /** + * Constructor overloads: + * 1. New pattern: (rpc, chainId, signer, fallbackRPCs?) - signer provided by KeyManager + * 2. Old pattern: (rpc, chainId, config, fallbackRPCs?) - for backward compatibility + */ public constructor( + keyManager: KeyManager, rpc: string, chainId: number, - config: OceanNodeConfig, fallbackRPCs?: string[] ) { - this.config = config this.chainId = chainId + this.keyManager = keyManager this.knownRPCs.push(rpc) if (fallbackRPCs && fallbackRPCs.length > 0) { this.knownRPCs.push(...fallbackRPCs) } + this.provider = undefined as undefined as FallbackProvider + this.signer = undefined as unknown as Signer + } - // get cached provider if it exists - const providerKey = `${chainId}-${rpc}` - if (Blockchain.providers.has(providerKey)) { - this.provider = Blockchain.providers.get(providerKey) - } else { - this.provider = new ethers.JsonRpcProvider(rpc, null, { - staticNetwork: ethers.Network.from(chainId) - }) - Blockchain.providers.set(providerKey, this.provider) - } - this.registerForNetworkEvents() - - // always use this signer, not simply provider.getSigner(0) for instance (as we do on many tests) - // get cached signer if it exists - const privateKeyHash = sha512(Buffer.from(this.config.keys.privateKey.raw)) - const signerKey = `${chainId}-${privateKeyHash}` - if (Blockchain.signers.has(signerKey)) { - let cachedSigner = Blockchain.signers.get(signerKey) - if (cachedSigner.provider !== this.provider) { - cachedSigner = (cachedSigner as ethers.Wallet).connect(this.provider) - Blockchain.signers.set(signerKey, cachedSigner) - } - this.signer = cachedSigner - } else { - this.signer = new ethers.Wallet( - Buffer.from(this.config.keys.privateKey.raw).toString('hex'), - this.provider - ) - Blockchain.signers.set(signerKey, this.signer) - } + public getSupportedChain(): number { + return this.chainId } - public getSigner(): Signer { - return this.signer + public getWallet(): Wallet { + return this.keyManager.getEthWallet() } - public getProvider(): JsonRpcApiProvider { - return this.provider + public async getWalletAddress(): Promise { + return await this.signer.getAddress() } - public getSupportedChain(): number { - return this.chainId + public async getProvider(force: boolean = false): Promise { + if (!this.provider) { + for (const rpc of this.knownRPCs) { + const rpcProvider = new JsonRpcProvider(rpc) + // filter wrong chains or broken RPCs + if (!force) { + try { + const { chainId } = await rpcProvider.getNetwork() + if (chainId.toString() === this.chainId.toString()) { + this.providers.push(rpcProvider) + break + } + } catch (error) { + CORE_LOGGER.error(`Error getting network for RPC ${rpc}: ${error}`) + } + } else { + this.providers.push(new JsonRpcProvider(rpc)) + } + } + this.provider = new FallbackProvider(this.providers) + } + return this.provider } - public async getWalletAddress(): Promise { - return await this.signer.getAddress() + public async getSigner(): Promise { + if (!this.signer) { + if (!this.provider) { + await this.getProvider() + } + this.signer = await this.keyManager.getEvmSigner(this.provider, this.chainId) + } + return this.signer } public async isNetworkReady(): Promise { - if (this.networkAvailable && this.provider.ready) { - return { ready: true } - } return await this.detectNetwork() } @@ -110,7 +113,7 @@ export class Blockchain { } public async calculateGasCost(to: string, amount: bigint): Promise { - const provider = this.getProvider() + const provider = await this.getProvider() const estimatedGas = await provider.estimateGas({ to, value: amount @@ -126,11 +129,11 @@ export class Blockchain { } public async sendTransaction( - wallet: Wallet, + signer: Signer, to: string, amount: bigint ): Promise { - const tx = await wallet.sendTransaction({ + const tx = await signer.sendTransaction({ to, value: amount }) @@ -139,15 +142,15 @@ export class Blockchain { return receipt } - private detectNetwork(): Promise { + private async detectNetwork(): Promise { + const provider = await this.getProvider() return new Promise((resolve) => { const timeout = setTimeout(() => { // timeout, hanging or invalid connection CORE_LOGGER.error(`Unable to detect provider network: (TIMEOUT)`) resolve({ ready: false, error: 'TIMEOUT' }) }, 3000) - - this.provider + provider .getBlock('latest') .then((block) => { clearTimeout(timeout) @@ -161,31 +164,7 @@ export class Blockchain { }) } - // try other rpc options, if available - public async tryFallbackRPCs(): Promise { - let response: ConnectionStatus = { ready: false, error: '' } - // we also retry the original one again after all the fallbacks - for (let i = this.knownRPCs.length - 1; i >= 0; i--) { - this.provider.off('network') - CORE_LOGGER.warn(`Retrying new provider connection with RPC: ${this.knownRPCs[i]}`) - this.provider = new JsonRpcProvider(this.knownRPCs[i]) - this.signer = new ethers.Wallet( - Buffer.from(this.config.keys.privateKey.raw).toString('hex'), - this.provider - ) - // try them 1 by 1 and wait a couple of secs for network detection - this.registerForNetworkEvents() - await sleep(2000) - response = await this.isNetworkReady() - // return as soon as we have a valid one - if (response.ready) { - return response - } - } - return response - } - - private registerForNetworkEvents() { + /* private registerForNetworkEvents() { this.provider.on('network', this.networkChanged) } @@ -194,10 +173,11 @@ export class Blockchain { // event with a null oldNetwork along with the newNetwork. So, if the // oldNetwork exists, it represents a changing network this.networkAvailable = newNetwork instanceof Network - } + } */ public async getFairGasPrice(gasFeeMultiplier: number): Promise { - const price = (await this.signer.provider.getFeeData()).gasPrice + const signer = await this.getSigner() + const price = (await signer.provider.getFeeData()).gasPrice const x = BigInt(price.toString()) if (gasFeeMultiplier) { const res = BigInt(price.toString()) * BigInt(gasFeeMultiplier) diff --git a/src/utils/config/builder.ts b/src/utils/config/builder.ts index 11688190f..1c56bce04 100644 --- a/src/utils/config/builder.ts +++ b/src/utils/config/builder.ts @@ -221,10 +221,15 @@ export function buildMergedConfig(): OceanNodeConfig { throw new Error('Invalid PRIVATE_KEY') } - const keys = getPeerIdFromPrivateKey(privateKey) + // const keys = getPeerIdFromPrivateKey(privateKey) + // Set key provider type and raw private key for KeyManager + // Type will be validated by KeyManager when initializing + const over = { + privateKey + } const { env } = process - const envOverrides: Record = { keys } + const envOverrides: Record = { over } Object.assign(envOverrides, mapEnvToConfig(env, ENV_TO_CONFIG_MAPPING)) diff --git a/src/utils/config/constants.ts b/src/utils/config/constants.ts index f3b43c244..8f99b8bb4 100644 --- a/src/utils/config/constants.ts +++ b/src/utils/config/constants.ts @@ -1,4 +1,5 @@ export const ENV_TO_CONFIG_MAPPING = { + PRIVATE_KEY: 'keys.privateKey', INTERFACES: 'INTERFACES', DB_URL: 'DB_URL', DB_USERNAME: 'DB_USERNAME', diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index a81bd26ce..5c56eaaaf 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -52,11 +52,9 @@ export const AccessListContractSchema = z.preprocess( z.record(z.string(), z.array(z.string())).nullable() ) -export const OceanNodeKeysSchema = z.object({ - peerId: z.any().optional(), - publicKey: z.any().optional(), - privateKey: z.any().optional(), - ethAddress: z.string().optional() +export const OceanNodeConfigKeysSchema = z.object({ + privateKey: z.any().optional().nullable(), + type: z.string().optional().default('raw') }) export const DenyListSchema = z.object({ @@ -265,7 +263,7 @@ export const OceanNodeConfigSchema = z authorizedPublishers: addressArrayFromString.optional().default([]), authorizedPublishersList: jsonFromString(AccessListContractSchema).optional(), - keys: OceanNodeKeysSchema.optional(), + keys: OceanNodeConfigKeysSchema.optional(), INTERFACES: z.string().optional(), hasP2P: booleanFromString.optional().default(true), diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 4f4012761..632efb277 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -201,7 +201,7 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.CONFIG_PATH, required: false }, - PRIVATE_KEY: { name: 'PRIVATE_KEY', value: process.env.PRIVATE_KEY, required: true }, + PRIVATE_KEY: { name: 'PRIVATE_KEY', value: process.env.PRIVATE_KEY, required: true }, // required for now as we support only raw private keys // used on test environments (ci) NODE1_PRIVATE_KEY: { name: 'NODE1_PRIVATE_KEY', diff --git a/src/utils/crypt.ts b/src/utils/crypt.ts index 8912bac05..694fb836b 100644 --- a/src/utils/crypt.ts +++ b/src/utils/crypt.ts @@ -1,62 +1,6 @@ -import eciesjs from 'eciesjs' -import crypto from 'crypto' -import { getConfiguration } from './config.js' import { EncryptMethod } from '../@types/fileObject.js' +import crypto from 'crypto' -/** - * This method encrypts data according to a given algorithm using node keys - * @param data data to encrypt - * @param algorithm encryption algorithm AES or ECIES - */ -export async function encrypt( - data: Uint8Array, - algorithm: EncryptMethod -): Promise { - let encryptedData: Buffer - const config = await getConfiguration() - const { privateKey, publicKey } = config.keys - if (algorithm === EncryptMethod.AES) { - // use first 16 bytes of public key as an initialisation vector - const initVector = publicKey.subarray(0, 16) - // creates cipher object, with the given algorithm, key and initialization vector - const cipher = crypto.createCipheriv('aes-256-cbc', privateKey.raw, initVector) - // encoding is ignored because we are working with bytes and want to return a buffer - encryptedData = Buffer.concat([cipher.update(data), cipher.final()]) - } else if (algorithm === EncryptMethod.ECIES) { - const sk = new eciesjs.PrivateKey(privateKey.raw) - // get public key from Elliptic curve - encryptedData = eciesjs.encrypt(sk.publicKey.toHex(), data) - } - return encryptedData -} - -/** - * This method decrypts data according to a given algorithm using node keys - * @param data data to decrypt - * @param algorithm decryption algorithm AES or ECIES - */ -export async function decrypt( - data: Uint8Array, - algorithm: EncryptMethod -): Promise { - let decryptedData: Buffer - const config = await getConfiguration() - const { privateKey, publicKey } = config.keys - if (algorithm === EncryptMethod.AES) { - // use first 16 bytes of public key as an initialisation vector - const initVector = publicKey.subarray(0, 16) - // creates decipher object, with the given algorithm, key and initialization vector - - const decipher = crypto.createDecipheriv('aes-256-cbc', privateKey.raw, initVector) - - // encoding is ignored because we are working with bytes and want to return a buffer - decryptedData = Buffer.concat([decipher.update(data), decipher.final()]) - } else if (algorithm === EncryptMethod.ECIES) { - const sk = new eciesjs.PrivateKey(privateKey.raw) - decryptedData = eciesjs.decrypt(sk.secret, data) - } - return decryptedData -} // this can be handy as we do this kind of hash in multiple places export function create256Hash(input: string): string { const result = crypto.createHash('sha256').update(input).digest('hex') diff --git a/src/utils/file.ts b/src/utils/file.ts index 15f27ee76..79998c763 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -7,7 +7,6 @@ import { import { OceanNode } from '../OceanNode.js' import { FindDdoHandler } from '../components/core/handler/ddoHandler.js' import { AssetUtils } from './asset.js' -import { decrypt } from './crypt.js' import { CORE_LOGGER } from './logging/common.js' import { sanitizeServiceFiles } from './util.js' import { isOrderingAllowedForAsset } from '../components/core/handler/downloadHandler.js' @@ -39,10 +38,12 @@ export async function getFile( throw new Error(msg) } // 3. Decrypt the url - const decryptedUrlBytes = await decrypt( - Uint8Array.from(Buffer.from(sanitizeServiceFiles(service.files), 'hex')), - EncryptMethod.ECIES - ) + const decryptedUrlBytes = await node + .getKeyManager() + .decrypt( + Uint8Array.from(Buffer.from(sanitizeServiceFiles(service.files), 'hex')), + EncryptMethod.ECIES + ) CORE_LOGGER.logMessage(`URL decrypted for Service ID: ${serviceId}`) // Convert the decrypted bytes back to a string