diff --git a/lib/routes/veeam/get.js b/lib/routes/veeam/get.js
index d802a29dfb..213d5c4866 100644
--- a/lib/routes/veeam/get.js
+++ b/lib/routes/veeam/get.js
@@ -1,9 +1,6 @@
-const xml2js = require('xml2js');
const { errors } = require('arsenal');
-const metadata = require('../../metadata/wrapper');
-const { respondWithData, buildHeadXML, getFileToBuild, isSystemXML } = require('./utils');
+const { respondWithData, buildHeadXML, buildVeeamFileData } = require('./utils');
const { responseXMLBody } = require('arsenal/build/lib/s3routes/routesUtils');
-const UtilizationService = require('../../../lib/utilization/instance');
/**
* Returns system.xml or capacity.xml files for a given bucket.
@@ -14,75 +11,22 @@ const UtilizationService = require('../../../lib/utilization/instance');
* @param {object} log - logger object
* @returns {undefined} -
*/
-function getVeeamFile(request, response, bucketMd, log) {
+async function getVeeamFile(request, response, bucketMd, log) {
if (!bucketMd) {
return responseXMLBody(errors.NoSuchBucket, null, response, log);
}
+
if ('tagging' in request.query) {
- return respondWithData(request, response, log, bucketMd,
+ return await respondWithData(request, response, log, bucketMd,
buildHeadXML(''));
}
- return metadata.getBucket(request.bucketName, log, (err, data) => {
- if (err) {
- return responseXMLBody(errors.InternalError, null, response, log);
- }
- const finalizeRequest = bucketMetrics => {
- const fileToBuild = getFileToBuild(request, data._capabilities?.VeeamSOSApi);
- if (fileToBuild.error) {
- return responseXMLBody(fileToBuild.error, null, response, log);
- }
-
- // Extract the last modified date, but do not include it when computing
- // the file's ETag (md5)
- const modified = fileToBuild.value.LastModified;
- delete fileToBuild.value.LastModified;
- // The SOSAPI metrics are dynamically computed using the SUR backend.
- if (bucketMetrics && !fileToBuild.value.CapacityInfo?.Used) {
- fileToBuild.value.CapacityInfo.Used = Number(bucketMetrics.bytesTotal);
- fileToBuild.value.CapacityInfo.Available =
- Number(fileToBuild.value.CapacityInfo.Capacity) - Number(bucketMetrics.bytesTotal);
- // TODO CLDSRV-633 when SUR backend supports realtime metrics: it will
- // report the real last cseq/date processed by SUR, instead of the current date,
- // ensuring no issue in a SOSAPI context. We should use this information.
- }
-
- const builder = new xml2js.Builder({
- headless: true,
- });
- return respondWithData(request, response, log, data,
- buildHeadXML(builder.buildObject(fileToBuild.value)), modified);
- };
- if (!isSystemXML(request.objectKey)) {
- const bucketKey = `${bucketMd._name}_${new Date(bucketMd._creationDate).getTime()}`;
- return UtilizationService.getUtilizationMetrics('bucket', bucketKey, null, {}, (err, bucketMetrics) => {
- if (err) {
- // Handle errors from UtilizationService/scubaclient
- // axios errors have status in err.response.status
- const statusCode = err.response?.status || err.statusCode || err.code;
- // Only handle 404 gracefully (no metrics available yet, e.g. post-install)
- // For 404, continue with static capacity data (Used=0 from bucket metadata)
- if (statusCode === 404) {
- log.warn('UtilizationService returned 404 when fetching capacity metrics', {
- method: 'getVeeamFile',
- bucket: request.bucketName,
- error: err.message || err.code,
- });
- return finalizeRequest();
- }
- log.error('error fetching capacity metrics from UtilizationService', {
- method: 'getVeeamFile',
- bucket: request.bucketName,
- error: err.message || err.code,
- statusCode,
- });
- return responseXMLBody(errors.InternalError, null, response, log);
- }
- return finalizeRequest(bucketMetrics);
- });
- }
- return finalizeRequest();
- });
+ try {
+ const result = await buildVeeamFileData(request, bucketMd, log);
+ return await respondWithData(request, response, log, result.bucketData, result.xmlContent, result.modified);
+ } catch (err) {
+ return responseXMLBody(err, null, response, log);
+ }
}
module.exports = getVeeamFile;
diff --git a/lib/routes/veeam/head.js b/lib/routes/veeam/head.js
index b94624b364..5aeca5f78e 100644
--- a/lib/routes/veeam/head.js
+++ b/lib/routes/veeam/head.js
@@ -1,7 +1,5 @@
-const xml2js = require('xml2js');
const { errors } = require('arsenal');
-const metadata = require('../../metadata/wrapper');
-const { getResponseHeader, buildHeadXML, getFileToBuild } = require('./utils');
+const { getResponseHeader, buildVeeamFileData } = require('./utils');
const { responseXMLBody, responseContentHeaders } = require('arsenal/build/lib/s3routes/routesUtils');
/**
@@ -13,31 +11,23 @@ const { responseXMLBody, responseContentHeaders } = require('arsenal/build/lib/s
* @param {object} log - logger object
* @returns {undefined} -
*/
-function headVeeamFile(request, response, bucketMd, log) {
+async function headVeeamFile(request, response, bucketMd, log) {
if (!bucketMd) {
return responseXMLBody(errors.NoSuchBucket, null, response, log);
}
- return metadata.getBucket(request.bucketName, log, (err, data) => {
- if (err) {
- return responseXMLBody(errors.InternalError, null, response, log);
- }
- const fileToBuild = getFileToBuild(request, data._capabilities?.VeeamSOSApi);
- if (fileToBuild.error) {
- return responseXMLBody(fileToBuild.error, null, response, log);
- }
- let modified = new Date().toISOString();
- // Extract the last modified date, but do not include it when computing
- // the file's ETag (md5)
- modified = fileToBuild.value.LastModified;
- delete fileToBuild.value.LastModified;
- // Recompute file content to generate appropriate content-md5 header
- const builder = new xml2js.Builder({
- headless: true,
- });
- const dataBuffer = Buffer.from(buildHeadXML(builder.buildObject(fileToBuild)));
- return responseContentHeaders(null, {}, getResponseHeader(request, data,
- dataBuffer, modified, log), response, log);
- });
+
+ try {
+ const result = await buildVeeamFileData(request, bucketMd, log);
+ return responseContentHeaders(
+ null,
+ {},
+ getResponseHeader(request, result.bucketData, result.dataBuffer, result.modified, log),
+ response,
+ log,
+ );
+ } catch (err) {
+ return responseXMLBody(err, null, response, log);
+ }
}
module.exports = headVeeamFile;
diff --git a/lib/routes/veeam/list.js b/lib/routes/veeam/list.js
index 38fd0293ee..175fbb77b4 100644
--- a/lib/routes/veeam/list.js
+++ b/lib/routes/veeam/list.js
@@ -1,13 +1,10 @@
const url = require('url');
-const xml2js = require('xml2js');
const { errors, errorInstances } = require('arsenal');
const querystring = require('querystring');
-const metadata = require('../../metadata/wrapper');
const { responseXMLBody } = require('arsenal/build/lib/s3routes/routesUtils');
-const { respondWithData, getResponseHeader, buildHeadXML, validPath } = require('./utils');
+const { respondWithData, getResponseHeader, buildXML, validPath, fetchCapacityMetrics, getBucket } = require('./utils');
const { processVersions, processMasterVersions } = require('../../api/bucketGet');
-
/**
* Utility function to build a standard response for the LIST route.
* It adds the supported path by default as a static and default file.
@@ -83,51 +80,67 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) {
* @param {object} log - logger object
* @returns {undefined} -
*/
-function listVeeamFiles(request, response, bucketMd, log) {
+async function listVeeamFiles(request, response, bucketMd, log) {
if (!bucketMd) {
return responseXMLBody(errors.NoSuchBucket, null, response, log);
}
+
// Only accept list-type query parameter
if (!('list-type' in request.query) && !('versions' in request.query)) {
return responseXMLBody(errorInstances.InvalidRequest
.customizeDescription('The Veeam folder does not support this action.'), null, response, log);
}
- return metadata.getBucket(request.bucketName, log, (err, data) => {
- if (err) {
+
+ let data;
+ try {
+ data = await getBucket(request.bucketName, log);
+ } catch {
+ return responseXMLBody(errors.InternalError, null, response, log);
+ }
+
+ let bucketMetrics;
+ if (data._capabilities?.VeeamSOSApi?.CapacityInfo) {
+ try {
+ bucketMetrics = await fetchCapacityMetrics(bucketMd, request, log);
+ } catch {
return responseXMLBody(errors.InternalError, null, response, log);
}
- const filesToBuild = [];
- const fieldsToGenerate = [];
- if (data._capabilities?.VeeamSOSApi?.SystemInfo) {
- fieldsToGenerate.push({
- ...data._capabilities?.VeeamSOSApi?.SystemInfo,
- name: `${validPath}system.xml`,
- });
- }
- if (data._capabilities?.VeeamSOSApi?.CapacityInfo) {
- fieldsToGenerate.push({
- ...data._capabilities?.VeeamSOSApi?.CapacityInfo,
- name: `${validPath}capacity.xml`,
- });
- }
- fieldsToGenerate.forEach(file => {
- const lastModified = file.LastModified;
- // eslint-disable-next-line no-param-reassign
- delete file.LastModified;
- const builder = new xml2js.Builder({
- headless: true,
- });
- const dataBuffer = Buffer.from(buildHeadXML(builder.buildObject(file)));
- filesToBuild.push({
- ...getResponseHeader(request, data,
- dataBuffer, lastModified, log),
- name: file.name,
- });
+ } else {
+ bucketMetrics = { date: new Date() };
+ }
+
+ const filesToBuild = [];
+ const fieldsToGenerate = [];
+ if (data._capabilities?.VeeamSOSApi?.SystemInfo) {
+ fieldsToGenerate.push({
+ ...data._capabilities?.VeeamSOSApi?.SystemInfo,
+ name: `${validPath}system.xml`,
+ });
+ }
+ if (data._capabilities?.VeeamSOSApi?.CapacityInfo) {
+ fieldsToGenerate.push({
+ ...data._capabilities?.VeeamSOSApi?.CapacityInfo,
+ name: `${validPath}capacity.xml`,
+ });
+ }
+ fieldsToGenerate.forEach(file => {
+ const isCapacity = file.name.endsWith('capacity.xml');
+ const lastModified = isCapacity
+ ? bucketMetrics.date
+ : file.LastModified;
+ // eslint-disable-next-line no-param-reassign
+ delete file.LastModified;
+ const dataBuffer = Buffer.from(buildXML(file));
+ filesToBuild.push({
+ ...getResponseHeader(request, data,
+ dataBuffer, lastModified, log),
+ name: file.name,
});
- // When `versions` is present, listing should return a versioned list
- return respondWithData(request, response, log, data,
- buildXMLResponse(request, filesToBuild, 'versions' in request.query));
});
+
+ // When `versions` is present, listing should return a versioned list
+ return await respondWithData(request, response, log, data,
+ buildXMLResponse(request, filesToBuild, 'versions' in request.query));
}
module.exports = listVeeamFiles;
diff --git a/lib/routes/veeam/put.js b/lib/routes/veeam/put.js
index 521d0d7d59..0e15d86051 100644
--- a/lib/routes/veeam/put.js
+++ b/lib/routes/veeam/put.js
@@ -1,4 +1,5 @@
const async = require('async');
+const { callbackify } = require('util');
const { parseString } = require('xml2js');
const { receiveData, isSystemXML, getFileToBuild } = require('./utils');
const { s3routes, errors } = require('arsenal');
@@ -28,7 +29,7 @@ function putVeeamFile(request, response, bucketMd, log) {
next => {
// Extract the data from the request, keep it in memory
writeContinue(request, response);
- return receiveData(request, log, next);
+ return callbackify(receiveData)(request, log, next);
},
(value, next) => parseString(value, { explicitArray: false }, (err, parsed) => {
// Convert the received XML to a JS object
diff --git a/lib/routes/veeam/utils.js b/lib/routes/veeam/utils.js
index 5ab082c6d5..3676669435 100644
--- a/lib/routes/veeam/utils.js
+++ b/lib/routes/veeam/utils.js
@@ -1,9 +1,17 @@
+const xml2js = require('xml2js');
const { errors, errorInstances, jsutil } = require('arsenal');
-const { Readable } = require('stream');
+const { Readable, Writable, pipeline: streamPipeline } = require('stream');
+const { promisify } = require('util');
const collectResponseHeaders = require('../../utilities/collectResponseHeaders');
const collectCorsHeaders = require('../../utilities/collectCorsHeaders');
const crypto = require('crypto');
const { prepareStream } = require('arsenal/build/lib/s3middleware/prepareStream');
+const UtilizationService = require('../../utilization/instance');
+const metadata = require('../../metadata/wrapper');
+
+const pipeline = promisify(streamPipeline);
+const getUtilizationMetrics = promisify((...args) => UtilizationService.getUtilizationMetrics(...args));
+const getBucket = promisify((...args) => metadata.getBucket(...args));
/**
* Decodes an URI and return the result.
@@ -20,43 +28,44 @@ function _decodeURI(uri) {
*
* @param {object} request - incoming request
* @param {object} log - logger object
- * @param {function} callback -
- * @returns {undefined}
+ * @returns {Promise}
*/
-function receiveData(request, log, callback) {
- // Get keycontent
+async function receiveData(request, log) {
const { parsedContentLength } = request;
const ContentLengthThreshold = 1024 * 1024; // 1MB
+
// Prevent memory overloads by limiting the size of the
// received data.
if (parsedContentLength > ContentLengthThreshold) {
- return callback(errorInstances.InvalidInput
- .customizeDescription(`maximum allowed content-length is ${ContentLengthThreshold} bytes`));
+ throw errorInstances.InvalidInput
+ .customizeDescription(`maximum allowed content-length is ${ContentLengthThreshold} bytes`);
}
- const value = Buffer.alloc(parsedContentLength);
- const cbOnce = jsutil.once(callback);
- const dataStream = prepareStream(request, request.streamingV4Params, log, cbOnce);
- let cursor = 0;
- let exceeded = false;
- dataStream.on('data', data => {
- if (cursor + data.length > parsedContentLength) {
- exceeded = true;
- }
- if (!exceeded) {
- data.copy(value, cursor);
- }
- cursor += data.length;
- });
- dataStream.on('end', () => {
- if (exceeded) {
- log.error('data stream exceed announced size',
- { parsedContentLength, overflow: cursor });
- return callback(errors.InternalError);
- } else {
- return callback(null, value.toString());
- }
+ return await new Promise((resolve, reject) => {
+ const settle = jsutil.once((err, result) => {
+ if (err) { return reject(err); }
+ return resolve(result);
+ });
+ let totalLength = 0;
+ const chunks = [];
+ const collector = new Writable({
+ write(chunk, _enc, cb) {
+ totalLength += chunk.length;
+ if (totalLength > parsedContentLength) {
+ log.error('data stream exceed announced size',
+ { parsedContentLength, overflow: totalLength });
+ return cb(errors.InternalError);
+ }
+ chunks.push(chunk);
+ return cb();
+ },
+ final(cb) {
+ settle(null, Buffer.concat(chunks).toString());
+ cb();
+ },
+ });
+ const dataStream = prepareStream(request, request.streamingV4Params, log, settle);
+ pipeline(dataStream, collector).catch(err => settle(err));
});
- return undefined;
}
/**
@@ -95,6 +104,17 @@ function getResponseHeader(request, bucket, dataBuffer, lastModified, log) {
responseMetaHeaders['x-amz-request-id'] = log.getSerializedUids();
return responseMetaHeaders;
}
+/**
+ * Builds a headless XML string wrapped in the standard SOSAPI XML declaration.
+ *
+ * @param {object} obj - JS object to serialize to XML
+ * @returns {string} formatted XML file content
+ */
+function buildXML(obj) {
+ const builder = new xml2js.Builder({ headless: true });
+ return buildHeadXML(builder.buildObject(obj));
+}
+
/**
* Generic function to respond to user with data using streams
*
@@ -104,23 +124,12 @@ function getResponseHeader(request, bucket, dataBuffer, lastModified, log) {
* @param {BucketInfo} bucket - bucket info
* @param {string} data - data to send
* @param {date} [lastModified] - last modified date of the value
- * @returns {undefined} -
+ * @returns {Promise} -
*/
-function respondWithData(request, response, log, bucket, data, lastModified) {
+async function respondWithData(request, response, log, bucket, data, lastModified) {
const dataBuffer = Buffer.from(data);
const responseMetaHeaders = getResponseHeader(request, bucket, dataBuffer, lastModified, log);
- response.on('finish', () => {
- let contentLength = 0;
- if (responseMetaHeaders && responseMetaHeaders['Content-Length']) {
- contentLength = responseMetaHeaders['Content-Length'];
- }
- log.end().addDefaultFields({ contentLength });
- log.end().info('responded with streamed content', {
- httpCode: response.statusCode,
- });
- });
-
if (responseMetaHeaders && typeof responseMetaHeaders === 'object') {
Object.keys(responseMetaHeaders).forEach(key => {
if (responseMetaHeaders[key] !== undefined) {
@@ -138,14 +147,26 @@ function respondWithData(request, response, log, bucket, data, lastModified) {
}
response.writeHead(200);
- const stream = Readable.from(dataBuffer);
- stream.pipe(response);
- stream.on('unpipe', () => {
- response.end();
- });
- stream.on('error', () => {
- response.end();
- });
+
+ let contentLength = 0;
+ if (responseMetaHeaders && responseMetaHeaders['Content-Length']) {
+ contentLength = responseMetaHeaders['Content-Length'];
+ }
+ log.end().addDefaultFields({ contentLength });
+
+ try {
+ // Use a single-element array so the Buffer is sent as one chunk rather
+ // than being iterated byte-by-byte by Readable.from.
+ await pipeline(Readable.from([dataBuffer]), response);
+ log.end().info('responded with streamed content', {
+ httpCode: response.statusCode,
+ });
+ } catch (err) {
+ log.end().error('error streaming response', {
+ httpCode: response.statusCode,
+ error: err.message,
+ });
+ }
}
const validPath = '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/';
@@ -172,31 +193,113 @@ function isSystemXML(objectKey) {
*/
function getFileToBuild(request, data, inlineLastModified = false) {
const _isSystemXML = isSystemXML(request.objectKey);
- const fileToBuild = _isSystemXML ? data?.SystemInfo
- : data?.CapacityInfo;
+ const fileToBuild = _isSystemXML ? data?.SystemInfo : data?.CapacityInfo;
+
if (!fileToBuild) {
return { error: errors.NoSuchKey };
}
+
const modified = fileToBuild.LastModified || (new Date()).toISOString();
const fieldName = _isSystemXML ? 'SystemInfo' : 'CapacityInfo';
+
if (inlineLastModified) {
fileToBuild.LastModified = modified;
- return {
- value: {
- [fieldName]: fileToBuild,
- },
- fieldName,
- };
} else {
delete fileToBuild.LastModified;
- return {
- value: {
- [fieldName]: fileToBuild,
- LastModified: modified,
- },
- fieldName,
- };
}
+
+ return {
+ value: {
+ [fieldName]: fileToBuild,
+ },
+ fieldName,
+ };
+}
+
+/**
+ * Fetches capacity metrics from UtilizationService for a bucket.
+ * Handles 404 gracefully (no metrics available yet, e.g. post-install),
+ * returning a default bucketMetrics with the current date so callers always
+ * receive a usable object.
+ *
+ * @param {object} bucketMd - bucket metadata
+ * @param {object} request - incoming request
+ * @param {object} log - logger object
+ * @returns {Promise