Skip to content

Commit d5799dd

Browse files
AlexisMoraAlexisMora
authored
Feat: Upload folder service (#430)
1 parent 282044d commit d5799dd

File tree

4 files changed

+351
-0
lines changed

4 files changed

+351
-0
lines changed

‎src/services/network/upload/upload-folder.service.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import { dirname } from 'node:path';
2+
import { isAlreadyExistsError } from '../../../utils/errors.utils';
3+
import { logger } from '../../../utils/logger.utils';
4+
import { DriveFolderService } from '../../drive/drive-folder.service';
5+
import { CreateFoldersParams, CreateFolderWithRetryParams, DELAYS_MS, MAX_RETRIES } from './upload.types';
6+
7+
export class UploadFolderService {
8+
static readonly instance = new UploadFolderService();
9+
async createFolders({
10+
foldersToCreate,
11+
destinationFolderUuid,
12+
currentProgress,
13+
emitProgress,
14+
}: CreateFoldersParams): Promise<Map<string, string>> {
15+
const folderMap = new Map<string, string>();
16+
for (const folder of foldersToCreate) {
17+
const parentPath = dirname(folder.relativePath);
18+
const parentUuid = parentPath === '.' ? destinationFolderUuid : folderMap.get(parentPath);
19+
20+
if (!parentUuid) {
21+
logger.warn(`Parent folder not found for ${folder.relativePath}, skipping...`);
22+
continue;
23+
}
24+
25+
const createdFolderUuid = await this.createFolderWithRetry({
26+
folderName: folder.name,
27+
parentFolderUuid: parentUuid,
28+
});
29+
30+
if (createdFolderUuid) {
31+
folderMap.set(folder.relativePath, createdFolderUuid);
32+
currentProgress.itemsUploaded++;
33+
emitProgress();
34+
}
35+
}
36+
return folderMap;
37+
}
38+
39+
async createFolderWithRetry({ folderName, parentFolderUuid }: CreateFolderWithRetryParams): Promise<string | null> {
40+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
41+
try {
42+
const [createFolderPromise] = DriveFolderService.instance.createFolder({
43+
plainName: folderName,
44+
parentFolderUuid,
45+
});
46+
47+
const createdFolder = await createFolderPromise;
48+
return createdFolder.uuid;
49+
} catch (error: unknown) {
50+
if (isAlreadyExistsError(error)) {
51+
logger.info(`Folder ${folderName} already exists, skipping...`);
52+
return null;
53+
}
54+
if (attempt < MAX_RETRIES) {
55+
const delay = DELAYS_MS[attempt];
56+
logger.warn(
57+
`Failed to create folder ${folderName},
58+
retrying in ${delay}ms... (attempt ${attempt + 1}/${MAX_RETRIES + 1})`,
59+
);
60+
await new Promise((resolve) => setTimeout(resolve, delay));
61+
} else {
62+
logger.error(`Failed to create folder ${folderName} after ${MAX_RETRIES + 1} attempts`);
63+
throw error;
64+
}
65+
}
66+
}
67+
return null;
68+
}
69+
}

‎src/services/network/upload/upload.types.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import { LoginUserDetails } from '../../../types/command.types';
2+
import { FileSystemNode } from '../../local-filesystem/local-filesystem.types';
3+
import { NetworkFacade } from '../network-facade.service';
4+
5+
export interface UploadResult {
6+
totalBytes: number;
7+
rootFolderId: string;
8+
uploadTimeMs: number;
9+
}
10+
11+
export interface UploadFolderHandlerParams {
12+
localPath: string;
13+
destinationFolderUuid: string;
14+
loginUserDetails: LoginUserDetails;
15+
jsonFlag?: boolean;
16+
onProgress: (progress: UploadProgress) => void;
17+
}
18+
19+
export interface UploadProgress {
20+
percentage: number;
21+
currentFile?: string;
22+
}
23+
24+
export interface CreateFoldersParams {
25+
foldersToCreate: FileSystemNode[];
26+
destinationFolderUuid: string;
27+
currentProgress: { itemsUploaded: number; bytesUploaded: number };
28+
emitProgress: () => void;
29+
}
30+
31+
export interface CreateFolderWithRetryParams {
32+
folderName: string;
33+
parentFolderUuid: string;
34+
}
35+
36+
export interface UploadFilesInBatchesParams {
37+
network: NetworkFacade;
38+
filesToUpload: FileSystemNode[];
39+
folderMap: Map<string, string>;
40+
bucket: string;
41+
destinationFolderUuid: string;
42+
currentProgress: { itemsUploaded: number; bytesUploaded: number };
43+
emitProgress: () => void;
44+
}
45+
46+
export interface UploadFileWithRetryParams {
47+
file: FileSystemNode;
48+
network: NetworkFacade;
49+
bucket: string;
50+
parentFolderUuid: string;
51+
}
52+
export const MAX_CONCURRENT_UPLOADS = 5;
53+
export const DELAYS_MS = [500, 1000, 2000];
54+
export const MAX_RETRIES = 2;

‎test/services/network/upload/upload-folder.service.test.ts‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest';
2+
import { DriveFolderService } from '../../../../src/services/drive/drive-folder.service';
3+
import { UploadFolderService } from '../../../../src/services/network/upload/upload-folder.service';
4+
import { logger } from '../../../../src/utils/logger.utils';
5+
import { isAlreadyExistsError } from '../../../../src/utils/errors.utils';
6+
import { createFileSystemNodeFixture, createProgressFixtures } from './upload.service.helpers';
7+
import { DELAYS_MS } from '../../../../src/services/network/upload/upload.types';
8+
vi.mock('../../../../src/services/drive/drive-folder.service', () => ({
9+
DriveFolderService: {
10+
instance: {
11+
createFolder: vi.fn(),
12+
},
13+
},
14+
}));
15+
16+
vi.mock('../../../../src/utils/logger.utils', () => ({
17+
logger: {
18+
warn: vi.fn(),
19+
info: vi.fn(),
20+
error: vi.fn(),
21+
},
22+
}));
23+
24+
vi.mock('../../../../src/utils/errors.utils', async (importOriginal) => {
25+
const actual = await importOriginal<typeof import('../../../../src/utils/errors.utils')>();
26+
return {
27+
...actual,
28+
isAlreadyExistsError: vi.fn(),
29+
ErrorUtils: {
30+
report: vi.fn(),
31+
},
32+
};
33+
});
34+
35+
describe('UploadFolderService', () => {
36+
let sut: UploadFolderService;
37+
beforeEach(() => {
38+
vi.clearAllMocks();
39+
sut = UploadFolderService.instance;
40+
vi.mocked(DriveFolderService.instance.createFolder).mockReturnValue([
41+
Promise.resolve({ uuid: 'mock-folder-uuid' }),
42+
] as unknown as ReturnType<typeof DriveFolderService.instance.createFolder>);
43+
vi.mocked(isAlreadyExistsError).mockReturnValue(false);
44+
});
45+
describe('createFolders', () => {
46+
const destinationFolderUuid = 'dest-uuid';
47+
48+
it('should properly return a map of created folders where key is relativePath and value is uuid', async () => {
49+
const { currentProgress, emitProgress } = createProgressFixtures();
50+
vi.mocked(DriveFolderService.instance.createFolder)
51+
.mockReturnValueOnce([Promise.resolve({ uuid: 'root-uuid' })] as unknown as ReturnType<
52+
typeof DriveFolderService.instance.createFolder
53+
>)
54+
.mockReturnValueOnce([Promise.resolve({ uuid: 'subfolder-uuid' })] as unknown as ReturnType<
55+
typeof DriveFolderService.instance.createFolder
56+
>);
57+
58+
const result = await sut.createFolders({
59+
foldersToCreate: [
60+
createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' }),
61+
createFileSystemNodeFixture({ type: 'folder', name: 'subfolder', relativePath: 'root/subfolder' }),
62+
],
63+
destinationFolderUuid,
64+
currentProgress,
65+
emitProgress,
66+
});
67+
68+
expect(result.size).toBe(2);
69+
expect(result.get('root')).toBe('root-uuid');
70+
expect(result.get('root/subfolder')).toBe('subfolder-uuid');
71+
});
72+
73+
it('should properly skip over folders that dont have a parentUuid', async () => {
74+
const { currentProgress, emitProgress } = createProgressFixtures();
75+
76+
const result = await sut.createFolders({
77+
foldersToCreate: [
78+
createFileSystemNodeFixture({ type: 'folder', name: 'orphan', relativePath: 'nonexistent/orphan' }),
79+
],
80+
destinationFolderUuid,
81+
currentProgress,
82+
emitProgress,
83+
});
84+
85+
expect(result.size).toBe(0);
86+
expect(logger.warn).toHaveBeenCalledWith('Parent folder not found for nonexistent/orphan, skipping...');
87+
expect(DriveFolderService.instance.createFolder).not.toHaveBeenCalled();
88+
});
89+
90+
it('should properly set as parent folder the destinationFolderUuid for base folder', async () => {
91+
const { currentProgress, emitProgress } = createProgressFixtures();
92+
93+
await sut.createFolders({
94+
foldersToCreate: [createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' })],
95+
destinationFolderUuid,
96+
currentProgress,
97+
emitProgress,
98+
});
99+
100+
expect(DriveFolderService.instance.createFolder).toHaveBeenCalledWith({
101+
plainName: 'root',
102+
parentFolderUuid: destinationFolderUuid,
103+
});
104+
});
105+
106+
it('should properly update the progress on successful folder creation', async () => {
107+
const { currentProgress, emitProgress } = createProgressFixtures();
108+
109+
await sut.createFolders({
110+
foldersToCreate: [createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' })],
111+
destinationFolderUuid,
112+
currentProgress,
113+
emitProgress,
114+
});
115+
116+
expect(currentProgress.itemsUploaded).toBe(1);
117+
expect(emitProgress).toHaveBeenCalledTimes(1);
118+
});
119+
});
120+
describe('createFolderWithRetry', () => {
121+
const folderName = 'test-folder';
122+
const parentFolderUuid = 'parent-uuid';
123+
124+
it('should properly create a folder and return the created folder uuid', async () => {
125+
vi.mocked(DriveFolderService.instance.createFolder).mockReturnValueOnce([
126+
Promise.resolve({ uuid: 'created-folder-uuid' }),
127+
] as unknown as ReturnType<typeof DriveFolderService.instance.createFolder>);
128+
129+
const result = await sut.createFolderWithRetry({ folderName, parentFolderUuid });
130+
131+
expect(result).toBe('created-folder-uuid');
132+
expect(DriveFolderService.instance.createFolder).toHaveBeenCalledWith({
133+
plainName: folderName,
134+
parentFolderUuid,
135+
});
136+
});
137+
138+
it('should properly return null if the folder already exists', async () => {
139+
const alreadyExistsError = new Error('Folder already exists');
140+
vi.mocked(isAlreadyExistsError).mockReturnValue(true);
141+
vi.mocked(DriveFolderService.instance.createFolder).mockReturnValueOnce([
142+
Promise.reject(alreadyExistsError),
143+
] as unknown as ReturnType<typeof DriveFolderService.instance.createFolder>);
144+
145+
const result = await sut.createFolderWithRetry({ folderName, parentFolderUuid });
146+
147+
expect(result).toBeNull();
148+
expect(logger.info).toHaveBeenCalledWith(`Folder ${folderName} already exists, skipping...`);
149+
});
150+
151+
it('should properly retry up to 3 times when error caught', async () => {
152+
vi.useFakeTimers();
153+
const transientError = new Error('Network error');
154+
155+
// Create promises with catch handlers to prevent unhandled rejections
156+
const rejection1 = Promise.reject(transientError).catch(() => {});
157+
const rejection2 = Promise.reject(transientError).catch(() => {});
158+
159+
vi.mocked(DriveFolderService.instance.createFolder)
160+
.mockReturnValueOnce([rejection1] as unknown as ReturnType<typeof DriveFolderService.instance.createFolder>)
161+
.mockReturnValueOnce([rejection2] as unknown as ReturnType<typeof DriveFolderService.instance.createFolder>)
162+
.mockReturnValueOnce([Promise.resolve({ uuid: 'success-uuid' })] as unknown as ReturnType<
163+
typeof DriveFolderService.instance.createFolder
164+
>);
165+
166+
const resultPromise = sut.createFolderWithRetry({ folderName, parentFolderUuid });
167+
168+
await vi.advanceTimersByTimeAsync(DELAYS_MS[0]);
169+
await vi.advanceTimersByTimeAsync(DELAYS_MS[1]);
170+
171+
const result = await resultPromise;
172+
173+
expect(result).toBe('success-uuid');
174+
expect(DriveFolderService.instance.createFolder).toHaveBeenCalledTimes(3);
175+
expect(logger.warn).toHaveBeenCalledTimes(2);
176+
177+
vi.useRealTimers();
178+
});
179+
180+
it('should properly throw an error when multiple unsuccessfull retries', async () => {
181+
const unhandledRejectionListener = vi.fn();
182+
process.on('unhandledRejection', unhandledRejectionListener);
183+
onTestFinished(() => {
184+
process.off('unhandledRejection', unhandledRejectionListener);
185+
});
186+
vi.useFakeTimers();
187+
vi.mocked(DriveFolderService.instance.createFolder).mockImplementation(() => {
188+
return [Promise.reject(new Error('Persistent network error'))] as unknown as ReturnType<
189+
typeof DriveFolderService.instance.createFolder
190+
>;
191+
});
192+
193+
const resultPromise = sut.createFolderWithRetry({ folderName, parentFolderUuid });
194+
await vi.advanceTimersByTimeAsync(DELAYS_MS[0]);
195+
await vi.advanceTimersByTimeAsync(DELAYS_MS[1]);
196+
await expect(resultPromise).rejects.toThrow('Persistent network error');
197+
expect(DriveFolderService.instance.createFolder).toHaveBeenCalledTimes(3);
198+
expect(logger.warn).toHaveBeenCalledTimes(2);
199+
expect(logger.error).toHaveBeenCalledWith(`Failed to create folder ${folderName} after 3 attempts`);
200+
201+
vi.useRealTimers();
202+
});
203+
});
204+
});

‎test/services/network/upload/upload.service.helpers.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import { vi } from 'vitest';
2+
import { FileSystemNode } from '../../../../src/services/local-filesystem/local-filesystem.types';
3+
4+
export function createFileSystemNodeFixture({
5+
type,
6+
name,
7+
relativePath,
8+
size = 0,
9+
absolutePath = `/absolute/${relativePath}`,
10+
}: {
11+
type: 'file' | 'folder';
12+
name: string;
13+
relativePath: string;
14+
size?: number;
15+
absolutePath?: string;
16+
}) {
17+
return { type, name, relativePath, size, absolutePath } as FileSystemNode;
18+
}
19+
export function createProgressFixtures() {
20+
return {
21+
currentProgress: { itemsUploaded: 0, bytesUploaded: 0 },
22+
emitProgress: vi.fn(),
23+
};
24+
}

0 commit comments

Comments
 (0)