diff --git a/constants.js b/constants.js index 8e713bc40a..1337f593e8 100644 --- a/constants.js +++ b/constants.js @@ -279,6 +279,14 @@ const constants = { rateLimitDefaultConfigCacheTTL: 30000, // 30 seconds rateLimitDefaultBurstCapacity: 1, rateLimitCleanupInterval: 10000, // 10 seconds + // Supported attributes for the GetObjectAttributes 'x-amz-optional-attributes' header. + supportedGetObjectAttributes: new Set([ + 'StorageClass', + 'ObjectSize', + 'ObjectParts', + 'Checksum', + 'ETag', + ]), }; module.exports = constants; diff --git a/lib/api/api.js b/lib/api/api.js index 195441c3fe..058e63c2e6 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -56,6 +56,7 @@ const { objectDelete } = require('./objectDelete'); const objectDeleteTagging = require('./objectDeleteTagging'); const objectGet = require('./objectGet'); const objectGetACL = require('./objectGetACL'); +const objectGetAttributes = require('./objectGetAttributes.js'); const objectGetLegalHold = require('./objectGetLegalHold'); const objectGetRetention = require('./objectGetRetention'); const objectGetTagging = require('./objectGetTagging'); @@ -471,6 +472,7 @@ const api = { objectDeleteTagging, objectGet, objectGetACL, + objectGetAttributes, objectGetLegalHold, objectGetRetention, objectGetTagging, diff --git a/lib/api/apiUtils/object/parseAttributesHeader.js b/lib/api/apiUtils/object/parseAttributesHeader.js new file mode 100644 index 0000000000..685ac93884 --- /dev/null +++ b/lib/api/apiUtils/object/parseAttributesHeader.js @@ -0,0 +1,25 @@ +const { errorInstances } = require('arsenal'); +const { supportedGetObjectAttributes } = require('../../../../constants'); + +/** + * parseAttributesHeaders - Parse and validate the x-amz-object-attributes header + * @param {object} headers - request headers + * @returns {Set} - set of requested attribute names + * @throws {Error} - InvalidRequest if header is missing/empty, InvalidArgument if attribute is invalid + */ +function parseAttributesHeaders(headers) { + const attributes = headers['x-amz-object-attributes']?.split(',').map(attr => attr.trim()) ?? []; + if (attributes.length === 0) { + throw errorInstances.InvalidRequest.customizeDescription( + 'The x-amz-object-attributes header specifying the attributes to be retrieved is either missing or empty', + ); + } + + if (attributes.some(attr => !supportedGetObjectAttributes.has(attr))) { + throw errorInstances.InvalidArgument.customizeDescription('Invalid attribute name specified.'); + } + + return new Set(attributes); +} + +module.exports = parseAttributesHeaders; diff --git a/lib/api/objectGetAttributes.js b/lib/api/objectGetAttributes.js new file mode 100644 index 0000000000..ffec684cca --- /dev/null +++ b/lib/api/objectGetAttributes.js @@ -0,0 +1,170 @@ +const { promisify } = require('util'); +const { errors } = require('arsenal'); +const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); +const collectCorsHeaders = require('../utilities/collectCorsHeaders'); +const parseAttributesHeaders = require('./apiUtils/object/parseAttributesHeader'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); +const { checkExpectedBucketOwner } = require('./apiUtils/authorization/bucketOwner'); +const { pushMetric } = require('../utapi/utilities'); +const { getPartCountFromMd5 } = require('./apiUtils/object/partInfo'); + +const checkExpectedBucketOwnerPromise = promisify(checkExpectedBucketOwner); +const validateBucketAndObj = promisify(standardMetadataValidateBucketAndObj); + +const OBJECT_GET_ATTRIBUTES = 'objectGetAttributes'; + +/** + * buildXmlResponse - Build XML response for GetObjectAttributes + * @param {object} objMD - object metadata + * @param {Set} requestedAttrs - set of requested attribute names + * @returns {string} XML response + */ +function buildXmlResponse(objMD, requestedAttrs) { + const xml = []; + xml.push( + '', + '', + ); + + for (const attribute of requestedAttrs) { + switch (attribute) { + case 'ETag': + xml.push(`${objMD['content-md5']}`); + break; + case 'ObjectParts': { + const partCount = getPartCountFromMd5(objMD); + if (partCount) { + xml.push( + '', + `${partCount}`, + '', + ); + } + break; + } + case 'StorageClass': + xml.push(`${objMD['x-amz-storage-class']}`); + break; + case 'ObjectSize': + xml.push(`${objMD['content-length']}`); + break; + } + } + + xml.push(''); + return xml.join(''); +} + +/** + * objectGetAttributes - Retrieves all metadata from an object without returning the object itself + * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info + * @param {object} request - http request object + * @param {object} log - Werelogs logger + * @param {function} callback - callback optional to keep backward compatibility + * @returns {Promise} - { xml, responseHeaders } + * @throws {ArsenalError} NoSuchVersion - if versionId specified but not found + * @throws {ArsenalError} NoSuchKey - if object not found + * @throws {ArsenalError} MethodNotAllowed - if object is a delete marker + */ +async function objectGetAttributes(authInfo, request, log, callback) { + if (callback) { + return objectGetAttributes(authInfo, request, log) + .then(result => callback(null, result.xml, result.responseHeaders)) + .catch(err => callback(err, null, err.responseHeaders ?? {})); + } + + log.trace('processing request', { method: OBJECT_GET_ATTRIBUTES }); + const { bucketName, objectKey, headers, actionImplicitDenies } = request; + + const versionId = decodeVersionId(request.query); + if (versionId instanceof Error) { + log.debug('invalid versionId query', { + method: OBJECT_GET_ATTRIBUTES, + versionId: request.query.versionId, + error: versionId, + }); + throw versionId; + } + + const metadataValParams = { + authInfo, + bucketName, + objectKey, + versionId, + getDeleteMarker: true, + requestType: request.apiMethods || OBJECT_GET_ATTRIBUTES, + request, + }; + + let bucket, objectMD; + try { + ({ bucket, objectMD } = await validateBucketAndObj(metadataValParams, actionImplicitDenies, log)); + await checkExpectedBucketOwnerPromise(headers, bucket, log); + } catch (err) { + log.debug('error validating bucket and object', { + method: OBJECT_GET_ATTRIBUTES, + bucket: bucketName, + key: objectKey, + versionId, + error: err, + }); + throw err; + } + + const responseHeaders = collectCorsHeaders(headers.origin, request.method, bucket); + + if (!objectMD) { + log.debug('object not found', { + method: OBJECT_GET_ATTRIBUTES, + bucket: bucketName, + key: objectKey, + versionId, + }); + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + err.responseHeaders = responseHeaders; + throw err; + } + + responseHeaders['x-amz-version-id'] = getVersionIdResHeader(bucket.getVersioningConfiguration(), objectMD); + responseHeaders['Last-Modified'] = objectMD['last-modified'] && new Date(objectMD['last-modified']).toUTCString(); + + if (objectMD.isDeleteMarker) { + log.debug('attempt to get attributes of a delete marker', { + method: OBJECT_GET_ATTRIBUTES, + bucket: bucketName, + key: objectKey, + versionId, + }); + responseHeaders['x-amz-delete-marker'] = true; + const err = errors.MethodNotAllowed; + err.responseHeaders = responseHeaders; + throw err; + } + + const requestedAttrs = parseAttributesHeaders(headers); + + if (requestedAttrs.has('Checksum')) { + log.debug('Checksum attribute requested but not implemented', { + method: OBJECT_GET_ATTRIBUTES, + bucket: bucketName, + key: objectKey, + versionId, + }); + const err = errors.NotImplemented.customizeDescription('Checksum attribute is not implemented'); + err.responseHeaders = responseHeaders; + throw err; + } + + pushMetric(OBJECT_GET_ATTRIBUTES, log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD?.versionId, + location: objectMD?.dataStoreName, + }); + + const xml = buildXmlResponse(objectMD, requestedAttrs); + return { xml, responseHeaders }; +} + +module.exports = objectGetAttributes; diff --git a/package.json b/package.json index 956c80b806..495cbb7a8d 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/arsenal#8.3.3", + "arsenal": "git+https://github.com/scality/Arsenal#8.3.4", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/tests/functional/aws-node-sdk/test/object/objectGetAttributes.js b/tests/functional/aws-node-sdk/test/object/objectGetAttributes.js new file mode 100644 index 0000000000..8a665dc12e --- /dev/null +++ b/tests/functional/aws-node-sdk/test/object/objectGetAttributes.js @@ -0,0 +1,254 @@ +const assert = require('assert'); +const { + CreateBucketCommand, + DeleteBucketCommand, + DeleteObjectCommand, + PutObjectCommand, + GetObjectAttributesCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, +} = require('@aws-sdk/client-s3'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); + +const bucket = 'testbucket'; +const key = 'testobject'; +const body = 'hello world!'; +const expectedMD5 = 'fc3ff98e8c6a0d3087d515c0473f8677'; + +describe('objectGetAttributes', () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + + before(() => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + }); + + beforeEach(async () => { + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); + }); + + afterEach(async () => { + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + it('should fail with a wrong bucket owner header', async () => { + try { + await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + ExpectedBucketOwner: 'wrongAccountId', + })); + assert.fail('Expected AccessDenied error'); + } catch (err) { + assert.strictEqual(err.name, 'AccessDenied'); + assert.strictEqual(err.message, 'Access Denied'); + } + }); + + it('should fail because attributes header is missing', async () => { + try { + await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: [], + })); + assert.fail('Expected InvalidArgument error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidArgument'); + assert.strictEqual(err.message, 'Invalid attribute name specified.'); + } + }); + + it('should fail because attribute name is invalid', async () => { + try { + await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['InvalidAttribute'], + })); + assert.fail('Expected InvalidArgument error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidArgument'); + assert.strictEqual(err.message, 'Invalid attribute name specified.'); + } + }); + + it('should return NoSuchKey for non-existent object', async () => { + try { + await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: 'nonexistent', + ObjectAttributes: ['ETag'], + })); + assert.fail('Expected NoSuchKey error'); + } catch (err) { + assert.strictEqual(err.name, 'NoSuchKey'); + assert.strictEqual(err.message, 'The specified key does not exist.'); + } + }); + + it('should return all attributes', async () => { + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize'], + })); + + assert.strictEqual(data.ETag, expectedMD5); + assert.strictEqual(data.StorageClass, 'STANDARD'); + assert.strictEqual(data.ObjectSize, body.length); + assert.strictEqual(data.ObjectParts, undefined, "ObjectParts shouldn't be present for non-MPU object"); + assert(data.LastModified, 'LastModified should be present'); + }); + + it('should return ETag', async () => { + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + })); + + assert.strictEqual(data.ETag, expectedMD5); + }); + + it('should fail with NotImplemented when Checksum is requested', async () => { + try { + await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['Checksum'], + })); + assert.fail('Expected NotImplemented error'); + } catch (err) { + assert.strictEqual(err.name, 'NotImplemented'); + assert.strictEqual(err.message, 'Checksum attribute is not implemented'); + } + }); + + it("shouldn't return ObjectParts for non-MPU objects", async () => { + // Requesting only ObjectParts for a non-MPU object break AWS SDK v3 + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ObjectParts', 'ETag'], + })); + + assert.strictEqual(data.ObjectParts, undefined, "ObjectParts shouldn't be present"); + assert.strictEqual(data.ETag, expectedMD5); + }); + + it('should return StorageClass', async () => { + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['StorageClass'], + })); + + assert.strictEqual(data.StorageClass, 'STANDARD'); + }); + + it('should return ObjectSize', async () => { + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ObjectSize'], + })); + + assert.strictEqual(data.ObjectSize, body.length); + }); + + it('should return LastModified', async () => { + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + })); + + assert(data.LastModified, 'LastModified should be present'); + assert(data.LastModified instanceof Date, 'LastModified should be a Date'); + assert(!isNaN(data.LastModified.getTime()), 'LastModified should be a valid date'); + }); + }); +}); + +describe('Test get object attributes with multipart upload', () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + const mpuKey = 'mpuObject'; + const partSize = 5 * 1024 * 1024; // Minimum part size is 5MB + const partCount = 3; + + before(async () => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + + const createResult = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: mpuKey, + })); + const uploadId = createResult.UploadId; + + const partData = Buffer.alloc(partSize, 'a'); + const parts = []; + for (let i = 1; i <= partCount; i++) { + const uploadResult = await s3.send(new UploadPartCommand({ + Bucket: bucket, + Key: mpuKey, + PartNumber: i, + UploadId: uploadId, + Body: partData, + })); + parts.push({ PartNumber: i, ETag: uploadResult.ETag }); + } + + await s3.send(new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: mpuKey, + UploadId: uploadId, + MultipartUpload: { Parts: parts }, + })); + }); + + after(async () => { + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: mpuKey })); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + it('should return TotalPartsCount for MPU object', async () => { + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: mpuKey, + ObjectAttributes: ['ObjectParts'], + })); + + assert(data.ObjectParts, 'ObjectParts should be present'); + assert.strictEqual(data.ObjectParts.TotalPartsCount, partCount); + }); + + it('should return TotalPartsCount along with other attributes for MPU object', async () => { + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: mpuKey, + ObjectAttributes: ['ETag', 'ObjectParts', 'ObjectSize', 'StorageClass'], + })); + + assert(data.ETag, 'ETag should be present'); + assert(data.ETag.includes(`-${partCount}`), `ETag should indicate MPU with ${partCount} parts`); + assert(data.ObjectParts, 'ObjectParts should be present'); + assert.strictEqual(data.ObjectParts.TotalPartsCount, partCount); + assert.strictEqual(data.ObjectSize, partSize * partCount); + assert.strictEqual(data.StorageClass, 'STANDARD'); + }); + }); +}); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js b/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js new file mode 100644 index 0000000000..bae4e16a1c --- /dev/null +++ b/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js @@ -0,0 +1,137 @@ +const assert = require('assert'); +const { + CreateBucketCommand, + DeleteBucketCommand, + DeleteObjectCommand, + PutBucketVersioningCommand, + PutObjectCommand, + GetObjectAttributesCommand, +} = require('@aws-sdk/client-s3'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util.js'); + +const bucket = 'testbucket'; +const key = 'testobject'; +const body = 'hello world!'; +const expectedMD5 = 'fc3ff98e8c6a0d3087d515c0473f8677'; + +describe('Test get object attributes with versioning', () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + + before(() => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + }); + + beforeEach(async () => { + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + })); + }); + + afterEach(done => { + removeAllVersions({ Bucket: bucket }, err => { + if (err) { + return done(err); + } + return s3.send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); + }); + }); + + it('should return NoSuchVersion for non-existent versionId', async () => { + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + })); + + const fakeVersionId = '111111111111111111111111111111111111111175636f7270'; + + try { + await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + VersionId: fakeVersionId, + ObjectAttributes: ['ETag'], + })); + assert.fail('Expected NoSuchVersion error'); + } catch (err) { + assert.strictEqual(err.name, 'NoSuchVersion'); + assert.strictEqual( + err.message, + 'Indicates that the version ID specified in the request does not match an existing version.', + ); + } + }); + + it('should return MethodNotAllowed for delete marker', async () => { + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + })); + + await s3.send(new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + })); + + try { + await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + })); + assert.fail('Expected MethodNotAllowed error'); + } catch (err) { + assert.strictEqual(err.name, 'MethodNotAllowed'); + assert.strictEqual(err.message, 'The specified method is not allowed against this resource.'); + } + }); + + it('should return attributes for specific version', async () => { + const putResult = await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + })); + const versionId = putResult.VersionId; + + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + ObjectAttributes: ['ETag', 'ObjectSize'], + })); + + assert.strictEqual(data.ETag, expectedMD5); + assert.strictEqual(data.ObjectSize, body.length); + assert(data.LastModified, 'LastModified should be present'); + }); + + it('should return VersionId for versioned object', async () => { + const putResult = await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + })); + const versionId = putResult.VersionId; + + const data = await s3.send(new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + })); + + assert.strictEqual(data.VersionId, versionId); + }); + }); +}); diff --git a/tests/unit/api/apiUtils/object/parseAttributesHeader.js b/tests/unit/api/apiUtils/object/parseAttributesHeader.js new file mode 100644 index 0000000000..b722668880 --- /dev/null +++ b/tests/unit/api/apiUtils/object/parseAttributesHeader.js @@ -0,0 +1,211 @@ +const assert = require('assert'); + +const parseAttributesHeaders = require('../../../../../lib/api/apiUtils/object/parseAttributesHeader'); + +describe('parseAttributesHeaders', () => { + describe('missing or empty header', () => { + it('should throw InvalidRequest error when header is missing', () => { + const headers = {}; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidRequest, true); + assert.strictEqual( + err.description, + 'The x-amz-object-attributes header specifying the attributes to be retrieved is either missing or empty', + ); + return true; + }, + ); + }); + + it('should throw InvalidArgument error when header is empty string', () => { + const headers = { 'x-amz-object-attributes': '' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + + it('should throw InvalidArgument error when header contains only whitespace', () => { + const headers = { 'x-amz-object-attributes': ' ' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + + it('should throw InvalidArgument error when header contains only commas', () => { + const headers = { 'x-amz-object-attributes': ',,,' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + }); + + describe('invalid attribute names', () => { + it('should throw InvalidArgument error for single invalid attribute', () => { + const headers = { 'x-amz-object-attributes': 'InvalidAttribute' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + + it('should throw InvalidArgument error when one attribute is invalid among valid ones', () => { + const headers = { 'x-amz-object-attributes': 'ETag,InvalidAttribute,ObjectSize' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + + it('should throw InvalidArgument error for multiple invalid attributes', () => { + const headers = { 'x-amz-object-attributes': 'Invalid1,Invalid2' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + }); + + describe('valid attribute names', () => { + it('should return set with single valid attribute ETag', () => { + const headers = { 'x-amz-object-attributes': 'ETag' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.deepStrictEqual(result, new Set(['ETag'])); + }); + + it('should return set with single valid attribute StorageClass', () => { + const headers = { 'x-amz-object-attributes': 'StorageClass' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.deepStrictEqual(result, new Set(['StorageClass'])); + }); + + it('should return set with single valid attribute ObjectSize', () => { + const headers = { 'x-amz-object-attributes': 'ObjectSize' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.deepStrictEqual(result, new Set(['ObjectSize'])); + }); + + it('should return set with single valid attribute ObjectParts', () => { + const headers = { 'x-amz-object-attributes': 'ObjectParts' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.deepStrictEqual(result, new Set(['ObjectParts'])); + }); + + it('should return set with single valid attribute Checksum', () => { + const headers = { 'x-amz-object-attributes': 'Checksum' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.deepStrictEqual(result, new Set(['Checksum'])); + }); + + it('should return set with multiple valid attributes', () => { + const headers = { 'x-amz-object-attributes': 'ETag,ObjectSize,StorageClass' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.deepStrictEqual(result, new Set(['ETag', 'ObjectSize', 'StorageClass'])); + }); + + it('should return set with all valid attributes', () => { + const headers = { 'x-amz-object-attributes': 'StorageClass,ObjectSize,ObjectParts,Checksum,ETag' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.strictEqual(result.size, 5); + assert(result.has('StorageClass')); + assert(result.has('ObjectSize')); + assert(result.has('ObjectParts')); + assert(result.has('Checksum')); + assert(result.has('ETag')); + }); + }); + + describe('whitespace handling', () => { + it('should trim whitespace around attribute names', () => { + const headers = { 'x-amz-object-attributes': ' ETag , ObjectSize ' }; + const result = parseAttributesHeaders(headers); + + assert(result instanceof Set); + assert.deepStrictEqual(result, new Set(['ETag', 'ObjectSize'])); + }); + + it('should throw InvalidArgument for extra commas between attributes', () => { + const headers = { 'x-amz-object-attributes': 'ETag,,ObjectSize' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + + it('should throw InvalidArgument for leading and trailing commas', () => { + const headers = { 'x-amz-object-attributes': ',ETag,ObjectSize,' }; + + assert.throws( + () => parseAttributesHeaders(headers), + err => { + assert(err.is); + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + return true; + }, + ); + }); + }); +}); diff --git a/tests/unit/api/objectGetAttributes.js b/tests/unit/api/objectGetAttributes.js new file mode 100644 index 0000000000..1f50bb9866 --- /dev/null +++ b/tests/unit/api/objectGetAttributes.js @@ -0,0 +1,489 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const { parseStringPromise } = require('xml2js'); + +const { bucketPut } = require('../../../lib/api/bucketPut'); +const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); +const DummyRequest = require('../DummyRequest'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const objectPut = require('../../../lib/api/objectPut'); +const { objectDelete } = require('../../../lib/api/objectDelete'); +const objectGetAttributes = require('../../../lib/api/objectGetAttributes'); +const objectPutPart = require('../../../lib/api/objectPutPart'); + +const log = new DummyRequestLogger(); +const authInfo = makeAuthInfo('accessKey1'); +const namespace = 'default'; +const bucketName = 'bucketname'; +const objectName = 'objectName'; +const body = 'hello world!'; +const postBody = Buffer.from(body, 'utf8'); +const expectedMD5 = 'fc3ff98e8c6a0d3087d515c0473f8677'; + +// Promisify helper for functions with non-standard callback signatures +const promisify = fn => (...args) => new Promise((resolve, reject) => { + fn(...args, (err, ...results) => { + if (err) { + reject(err); + } else { + resolve(results); + } + }); +}); + +const bucketPutAsync = promisify(bucketPut); +const bucketPutVersioningAsync = promisify(bucketPutVersioning); +const objectPutAsync = promisify(objectPut); +const objectDeleteAsync = promisify(objectDelete); +const initiateMultipartUploadAsync = promisify(initiateMultipartUpload); +const objectPutPartAsync = promisify(objectPutPart); +const completeMultipartUploadAsync = promisify(completeMultipartUpload); + +const testPutBucketRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${bucketName}`, + actionImplicitDenies: false, +}; + +const createGetAttributesRequest = (attributes, options = {}) => { + const key = options.objectKey || objectName; + return { + bucketName, + namespace, + objectKey: key, + headers: { + 'x-amz-object-attributes': attributes.join(','), + ...options.headers, + }, + url: `/${bucketName}/${key}`, + query: options.query || {}, + actionImplicitDenies: false, + }; +}; + +describe('objectGetAttributes API', () => { + beforeEach(async () => { + cleanup(); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': `${postBody.length}`, + }, + parsedContentLength: postBody.length, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); + await bucketPutAsync(authInfo, testPutBucketRequest, log); + await objectPutAsync(authInfo, testPutObjectRequest, undefined, log); + }); + + it('should fail because attributes header is missing', async () => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + query: {}, + actionImplicitDenies: false, + }; + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.InvalidRequest, true); + assert.strictEqual( + err.description, + 'The x-amz-object-attributes header specifying the attributes ' + + 'to be retrieved is either missing or empty', + ); + } + }); + + it('should fail because attributes header is empty', async () => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-attributes': '', + }, + url: `/${bucketName}/${objectName}`, + query: {}, + actionImplicitDenies: false, + }; + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + } + }); + + it('should fail because attribute name is invalid', async () => { + const testGetRequest = createGetAttributesRequest(['InvalidAttribute']); + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.InvalidArgument, true); + assert.strictEqual(err.description, 'Invalid attribute name specified.'); + } + }); + + it('should return NoSuchKey for non-existent object', async () => { + const testGetRequest = createGetAttributesRequest(['ETag'], { + objectKey: 'nonexistent', + }); + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.NoSuchKey, true); + assert.strictEqual(err.description, 'The specified key does not exist.'); + } + }); + + it('should fail because of bad bucket owner', async () => { + const testGetRequest = createGetAttributesRequest(['ETag'], { + headers: { + 'x-amz-expected-bucket-owner': 'wrongAccountId', + }, + }); + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.AccessDenied, true); + assert.strictEqual(err.description, 'Access Denied'); + } + }); + + it('should return all attributes', async () => { + const testGetRequest = createGetAttributesRequest([ + 'ETag', + 'ObjectParts', + 'StorageClass', + 'ObjectSize', + ]); + + const { xml, responseHeaders } = await objectGetAttributes(authInfo, testGetRequest, log); + assert(xml, 'Response XML should be present'); + assert(responseHeaders['Last-Modified'], 'Last-Modified header should be present'); + + const result = await parseStringPromise(xml); + const response = result.GetObjectAttributesResponse; + + assert.strictEqual(response.ETag[0], expectedMD5); + assert.strictEqual(response.StorageClass[0], 'STANDARD'); + assert.strictEqual(response.ObjectSize[0], String(body.length)); + assert.strictEqual(response.ObjectParts, undefined, "ObjectParts shouldn't be present for non-MPU object"); + assert(responseHeaders['Last-Modified'], 'LastModified should be present'); + }); + + it('should return ETag', async () => { + const testGetRequest = createGetAttributesRequest(['ETag']); + + const { xml } = await objectGetAttributes(authInfo, testGetRequest, log); + const result = await parseStringPromise(xml); + assert.strictEqual(result.GetObjectAttributesResponse.ETag[0], expectedMD5); + }); + + it('should fail with NotImplemented when Checksum is requested', async () => { + const testGetRequest = createGetAttributesRequest(['Checksum']); + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.NotImplemented, true); + assert.strictEqual(err.description, 'Checksum attribute is not implemented'); + } + }); + + it("shouldn't return ObjectParts for non-MPU object", async () => { + const testGetRequest = createGetAttributesRequest(['ObjectParts']); + + const { xml } = await objectGetAttributes(authInfo, testGetRequest, log); + const result = await parseStringPromise(xml); + assert.strictEqual( + result.GetObjectAttributesResponse.ObjectParts, + undefined, + "ObjectParts shouldn't be present", + ); + }); + + it('should return StorageClass', async () => { + const testGetRequest = createGetAttributesRequest(['StorageClass']); + + const { xml } = await objectGetAttributes(authInfo, testGetRequest, log); + const result = await parseStringPromise(xml); + assert.strictEqual(result.GetObjectAttributesResponse.StorageClass[0], 'STANDARD'); + }); + + it('should return ObjectSize', async () => { + const testGetRequest = createGetAttributesRequest(['ObjectSize']); + + const { xml } = await objectGetAttributes(authInfo, testGetRequest, log); + const result = await parseStringPromise(xml); + assert.strictEqual(result.GetObjectAttributesResponse.ObjectSize[0], String(body.length)); + }); + + it('should return LastModified in response headers', async () => { + const testGetRequest = createGetAttributesRequest(['ETag']); + + const { responseHeaders } = await objectGetAttributes(authInfo, testGetRequest, log); + assert(responseHeaders['Last-Modified'], 'Last-Modified should be present'); + assert(!isNaN(new Date(responseHeaders['Last-Modified']).getTime()), 'Last-Modified should be a valid date'); + }); +}); + +describe('objectGetAttributes API with multipart upload', () => { + const partCount = 2; + const partBody = Buffer.from('I am a part\n', 'utf8'); + + const createMpuObject = async () => { + const initiateRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectName}?uploads`, + actionImplicitDenies: false, + }; + + const [result] = await initiateMultipartUploadAsync(authInfo, initiateRequest, log); + const json = await parseStringPromise(result); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + + const completeParts = []; + for (let i = 1; i <= partCount; i++) { + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '5242880', + }, + parsedContentLength: 5242880, + url: `/${objectName}?partNumber=${i}&uploadId=${testUploadId}`, + query: { + partNumber: String(i), + uploadId: testUploadId, + }, + partHash, + }, + partBody, + ); + await objectPutPartAsync(authInfo, partRequest, undefined, log); + completeParts.push(`${i}"${partHash}"`); + } + + const completeBody = + `${completeParts.join('')}`; + + const completeRequest = { + bucketName, + namespace, + objectKey: objectName, + parsedHost: 's3.amazonaws.com', + url: `/${objectName}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + + await completeMultipartUploadAsync(authInfo, completeRequest, log); + }; + + beforeEach(async () => { + cleanup(); + await bucketPutAsync(authInfo, testPutBucketRequest, log); + await createMpuObject(); + }); + + it('should return TotalPartsCount for MPU object', async () => { + const testGetRequest = createGetAttributesRequest(['ObjectParts']); + + const { xml } = await objectGetAttributes(authInfo, testGetRequest, log); + const result = await parseStringPromise(xml); + const response = result.GetObjectAttributesResponse; + + assert(response.ObjectParts, 'ObjectParts should be present'); + assert.strictEqual(response.ObjectParts[0].PartsCount[0], String(partCount)); + }); + + it('should return TotalPartsCount along with other attributes for MPU object', async () => { + const testGetRequest = createGetAttributesRequest(['ETag', 'ObjectParts', 'ObjectSize', 'StorageClass']); + + const { xml } = await objectGetAttributes(authInfo, testGetRequest, log); + const result = await parseStringPromise(xml); + const response = result.GetObjectAttributesResponse; + + assert(response.ETag, 'ETag should be present'); + assert(response.ETag[0].includes(`-${partCount}`), `ETag should indicate MPU with ${partCount} parts`); + assert(response.ObjectParts, 'ObjectParts should be present'); + assert.strictEqual(response.ObjectParts[0].PartsCount[0], String(partCount)); + assert(response.ObjectSize, 'ObjectSize should be present'); + assert.strictEqual(response.StorageClass[0], 'STANDARD'); + }); +}); + +describe('objectGetAttributes API with versioning', () => { + const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); + + beforeEach(async () => { + cleanup(); + await bucketPutAsync(authInfo, testPutBucketRequest, log); + await bucketPutVersioningAsync(authInfo, enableVersioningRequest, log); + }); + + it('should return NoSuchVersion for non-existent versionId', async () => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': `${postBody.length}`, + }, + parsedContentLength: postBody.length, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); + + const fakeVersionId = '111111111111111111111111111111111111111175636f7270'; + + await objectPutAsync(authInfo, testPutObjectRequest, undefined, log); + const testGetRequest = createGetAttributesRequest(['ETag'], { + query: { versionId: fakeVersionId }, + }); + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.NoSuchVersion, true); + assert.strictEqual( + err.description, + 'Indicates that the version ID specified in the request does not match an existing version.', + ); + } + }); + + it('should return MethodNotAllowed for delete marker', async () => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': `${postBody.length}`, + }, + parsedContentLength: postBody.length, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); + + const testDeleteRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; + + await objectPutAsync(authInfo, testPutObjectRequest, undefined, log); + await objectDeleteAsync(authInfo, testDeleteRequest, log); + + const testGetRequest = createGetAttributesRequest(['ETag']); + + try { + await objectGetAttributes(authInfo, testGetRequest, log); + assert.fail('Expected error was not thrown'); + } catch (err) { + assert.strictEqual(err.is.MethodNotAllowed, true); + assert.strictEqual(err.description, 'The specified method is not allowed against this resource.'); + assert.strictEqual(err.responseHeaders['x-amz-delete-marker'], true); + } + }); + + it('should return attributes for specific version', async () => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': `${postBody.length}`, + }, + parsedContentLength: postBody.length, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); + + const [resHeaders] = await objectPutAsync(authInfo, testPutObjectRequest, undefined, log); + const versionId = resHeaders['x-amz-version-id']; + assert(versionId, 'Version ID should be present'); + + const testGetRequest = createGetAttributesRequest(['ETag', 'ObjectSize'], { + query: { versionId }, + }); + + const { xml, responseHeaders } = await objectGetAttributes(authInfo, testGetRequest, log); + assert(responseHeaders['Last-Modified'], 'Last-Modified should be present'); + + const result = await parseStringPromise(xml); + const response = result.GetObjectAttributesResponse; + + assert.strictEqual(response.ETag[0], expectedMD5); + assert.strictEqual(response.ObjectSize[0], String(body.length)); + }); + + it('should return VersionId in response headers for versioned object', async () => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': `${postBody.length}`, + }, + parsedContentLength: postBody.length, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); + + const [resHeaders] = await objectPutAsync(authInfo, testPutObjectRequest, undefined, log); + const versionId = resHeaders['x-amz-version-id']; + assert(versionId, 'Version ID should be present from PUT'); + + const testGetRequest = createGetAttributesRequest(['ETag']); + + const { responseHeaders } = await objectGetAttributes(authInfo, testGetRequest, log); + assert.strictEqual(responseHeaders['x-amz-version-id'], versionId); + }); +}); diff --git a/yarn.lock b/yarn.lock index 99ced2bebc..71f1438d19 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2551,6 +2551,26 @@ events "^3.0.0" tslib "^2.8.1" +"@azure/storage-blob@^12.31.0": + version "12.31.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.31.0.tgz#97b09be2bf6ab59739b862edd8124798362ce720" + integrity sha512-DBgNv10aCSxopt92DkTDD0o9xScXeBqPKGmR50FPZQaEcH4JLQ+GEOGEDv19V5BMkB7kxr+m4h6il/cCDPvmHg== + dependencies: + "@azure/abort-controller" "^2.1.2" + "@azure/core-auth" "^1.9.0" + "@azure/core-client" "^1.9.3" + "@azure/core-http-compat" "^2.2.0" + "@azure/core-lro" "^2.2.0" + "@azure/core-paging" "^1.6.2" + "@azure/core-rest-pipeline" "^1.19.1" + "@azure/core-tracing" "^1.2.0" + "@azure/core-util" "^1.11.0" + "@azure/core-xml" "^1.4.5" + "@azure/logger" "^1.1.4" + "@azure/storage-common" "^12.3.0" + events "^3.0.0" + tslib "^2.8.1" + "@azure/storage-common@^12.0.0-beta.2": version "12.0.0" resolved "https://registry.yarnpkg.com/@azure/storage-common/-/storage-common-12.0.0.tgz#a652d7daeb252b7827362b4e818f52fee15a1264" @@ -2566,6 +2586,21 @@ events "^3.3.0" tslib "^2.8.1" +"@azure/storage-common@^12.3.0": + version "12.3.0" + resolved "https://registry.yarnpkg.com/@azure/storage-common/-/storage-common-12.3.0.tgz#5bf257383836e67a426c91d7e9678479afe802a9" + integrity sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ== + dependencies: + "@azure/abort-controller" "^2.1.2" + "@azure/core-auth" "^1.9.0" + "@azure/core-http-compat" "^2.2.0" + "@azure/core-rest-pipeline" "^1.19.1" + "@azure/core-tracing" "^1.2.0" + "@azure/core-util" "^1.11.0" + "@azure/logger" "^1.1.4" + events "^3.3.0" + tslib "^2.8.1" + "@babel/code-frame@^7.26.2": version "7.26.2" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.26.2.tgz#4b5fab97d33338eff916235055f0ebc21e573a85" @@ -5349,16 +5384,16 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#8.3.3": - version "8.3.3" - resolved "git+https://github.com/scality/arsenal#d6120cfd139f23dd776785db92ba91a91063bba6" +"arsenal@git+https://github.com/scality/Arsenal#8.3.4": + version "8.3.4" + resolved "git+https://github.com/scality/Arsenal#b143ef75cde4ac796ef9e88ce90bcd6f7a575c56" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" "@aws-sdk/credential-providers" "^3.975.0" "@aws-sdk/lib-storage" "^3.975.0" "@azure/identity" "^4.13.0" - "@azure/storage-blob" "^12.28.0" + "@azure/storage-blob" "^12.31.0" "@js-sdsl/ordered-set" "^4.4.2" "@scality/hdclient" "^1.3.1" "@smithy/node-http-handler" "^4.3.0"