From e013b0d222f7c23786dd8993fc4b60a2d61fe8de Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Mon, 26 Jan 2026 14:53:16 +0100 Subject: [PATCH 1/3] use lighter logs to avoid unable to serialize errors ISSUE: S3UTILS-219 --- CountItems/CountManager.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CountItems/CountManager.js b/CountItems/CountManager.js index f4d4c1ae..34f0fc3e 100644 --- a/CountItems/CountManager.js +++ b/CountItems/CountManager.js @@ -44,7 +44,11 @@ class CountManager { this.log.info('processing a bucket', { method: 'CountManager::_setupQueue', workInQueue: this.q.length(), - bucketInfo, + bucketInfo: { + name: bucketInfo.getName(), + uid: bucketInfo.getUid(), + creationDate: bucketInfo.getCreationDate(), + } }); if (err) { return done(err); From 41b6cd07c9be4c8a86a54f7b4c8455a043363056 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Mon, 26 Jan 2026 17:48:47 +0100 Subject: [PATCH 2/3] optimize count items on bucket creation date query ISSUE: S3UTILS-219 --- tests/unit/utils/S3UtilsMongoClient.js | 54 ----------------- utils/S3UtilsMongoClient.js | 84 +++++++++++--------------- 2 files changed, 34 insertions(+), 104 deletions(-) diff --git a/tests/unit/utils/S3UtilsMongoClient.js b/tests/unit/utils/S3UtilsMongoClient.js index 8b225c38..1f1929a4 100644 --- a/tests/unit/utils/S3UtilsMongoClient.js +++ b/tests/unit/utils/S3UtilsMongoClient.js @@ -1026,60 +1026,6 @@ describe('S3UtilsMongoClient::_processEntryData', () => { })); }); -describe('S3UtilsMongoClient::_getUsersBucketCreationDates', () => { - let client; - let repl; - beforeAll(async () => { - repl = await MongoMemoryReplSet.create(mongoMemoryServerParams); - client = new S3UtilsMongoClient({ - ...createMongoParamsFromMongoMemoryRepl(repl), - logger, - }); - const setupPromise = util.promisify(client.setup).bind(client); - await setupPromise(); - }); - - afterAll(done => async.series([ - next => client.close(next), - next => repl.stop() - .then(() => next()) - .catch(next), - ], done)); - - it('should return empty array when no users bucket', async () => { - const testResults = await client._getUsersBucketCreationDates(logger); - assert.deepStrictEqual(testResults, {}); - }); - - it('should list existing buckets and return their creation dates', async () => { - const bucketName = 'bucket1'; - await new Promise((resolve, reject) => { - client.putObject( - USERSBUCKET, - `${testAccountCanonicalId}${constants.splitter}${bucketName}`, - testUserBucketInfo.value, - { - versioning: false, - versionId: null, - }, - logger, - err => { - if (err) { - reject(err); - } else { - resolve(); - } - }, - ); - }); - const testResults = await client._getUsersBucketCreationDates(logger); - const expectedRes = [ - `${testAccountCanonicalId}${constants.splitter}${bucketName}`, - ]; - assert.deepStrictEqual(Object.keys(testResults), expectedRes); - }); -}); - function createBucket(client, bucketName, isVersioned, callback) { const bucketMD = BucketInfo.fromObj({ ...testBucketMD, diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index ccd7803f..78df11e7 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -2,6 +2,7 @@ const { MongoClientInterface } = require('arsenal').storage.metadata.mongoclient const { Long } = require('mongodb'); const { errors, constants } = require('arsenal'); const async = require('async'); +const { promisify } = require('util'); const { validStorageMetricLevels } = require('../CountItems/utils/constants'); const getLocationConfig = require('./locationConfig'); const monitoring = require('./monitoring'); @@ -52,35 +53,6 @@ const baseMetricsObject = { }; class S3UtilsMongoClient extends MongoClientInterface { - /** - * Get the list of buckets and their location dates - * @param {object} log - Werelogs logger - * @returns {object} - Object with bucket names as keys - * and their creation dates as values - */ - async _getUsersBucketCreationDates(log) { - let cursorUsersBucketCreationDates; - try { - cursorUsersBucketCreationDates = await this.getCollection(USERSBUCKET).find({}, { - projection: { - 'value.creationDate': 1, - }, - }); - const usersBucketCreationDatesArray = await cursorUsersBucketCreationDates.toArray(); - return usersBucketCreationDatesArray - .reduce((map, obj) => ({ ...map, [obj._id]: obj.value.creationDate }), {}); - } catch (err) { - log.error('Failed to read __usersbucket collection', { - method: 'getUsersBucketCreationDates', - errDetails: { ...err }, - errorString: err.toString(), - }); - return null; - } finally { - await cursorUsersBucketCreationDates.close(); - } - } - async updateInflightDeltas(allMetrics, log) { let cursor; try { @@ -100,13 +72,11 @@ class S3UtilsMongoClient extends MongoClientInterface { const inflights = await cursor.toArray(); // convert inflights to a map with _id: usedCapacity._inflight - const inflightsMap = inflights.reduce((map, obj) => { - const inflightLong = obj.usedCapacity?._inflight || 0n; - return { - ...map, - [obj._id]: inflightLong, - }; - }, {}); + const inflightsMap = {}; + for (const inflight of inflights) { + const inflightValue = inflight.usedCapacity?._inflight || 0n; + inflightsMap[inflight._id] = inflightValue; + } const accountInflights = {}; allMetrics.forEach(entry => { @@ -201,22 +171,21 @@ class S3UtilsMongoClient extends MongoClientInterface { const locationConfig = getLocationConfig(log); - const usersBucketCreationDatesMap = await this._getUsersBucketCreationDates(log); + let bucketCreationDate; + const getUsersBucketCreationDateAsync = promisify(this.getUsersBucketCreationDate).bind(this); + try { + bucketCreationDate = await getUsersBucketCreationDateAsync(bucketInfo.getOwner(), bucketName, log); + } catch (err) { + return callback(errors.InternalError); + } const bucketStatus = bucketInfo.getVersioningConfiguration(); const isVer = (bucketStatus && (bucketStatus.Status === 'Enabled' || bucketStatus.Status === 'Suspended')); - if (!usersBucketCreationDatesMap) { - return callback(errors.InternalError); - } - - const bucketDate = usersBucketCreationDatesMap[`${bucketInfo.getOwner()}${constants.splitter}${bucketName}`]; - if (bucketDate) { - bucketKey = `bucket_${bucketName}_${new Date(bucketDate).getTime()}`; - if (bucketKey) { - inflightsPreScan = await this.readStorageConsumptionInflights(bucketKey, log); - } + if (bucketCreationDate) { + bucketKey = `bucket_${bucketName}_${new Date(bucketCreationDate).getTime()}`; + inflightsPreScan = await this.readStorageConsumptionInflights(bucketKey, log); } let startCursorDate = new Date(); @@ -243,7 +212,7 @@ class S3UtilsMongoClient extends MongoClientInterface { bucketName, bucketInfo, entry, - usersBucketCreationDatesMap[`${entry.value['owner-id']}${constants.splitter}${bucketName}`], + bucketCreationDate, isTransient, locationConfig, { @@ -1022,6 +991,19 @@ class S3UtilsMongoClient extends MongoClientInterface { } } + /** + * Read the bucket creation date from the `__usersbucket` collection. + * + * If the bucket entry is missing or does not contain a creation date, + * the function doesn't throw an error, and instead + * invoke the callback with no data `(null, undefined)`. + * + * @param {string} ownerId - Bucket owner canonical ID + * @param {string} bucketName - Bucket name + * @param {Object} log - Logger + * @param {Function} cb - Node-style callback: `(err, creationDate)` + * @returns {void} + */ async getUsersBucketCreationDate(ownerId, bucketName, log, cb) { try { const usersBucketCol = this.getCollection(USERSBUCKET); @@ -1033,15 +1015,17 @@ class S3UtilsMongoClient extends MongoClientInterface { }, }); if (!res || !res.value || !res.value.creationDate) { - log.error('bucket entry not found in __usersbucket', { + log.warn('bucket entry not found in __usersbucket', { + method: 'getUsersBucketCreationDate', bucketName, ownerId, }); - return cb(new Error('Bucket entry not found')); + return cb(null, undefined); } return cb(null, res.value.creationDate); } catch (err) { log.error('failed to read bucket entry from __usersbucket', { + method: 'getUsersBucketCreationDate', bucketName, ownerId, errDetails: { ...err }, From db0e5b328f8396a594d499054f5d6b2de699f45e Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Wed, 28 Jan 2026 13:03:47 +0100 Subject: [PATCH 3/3] bump version to 1.17.3 ISSUE: S3UTILS-219 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a4010179..538e6829 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.17.2", + "version": "1.17.3", "engines": { "node": ">= 22" },