BlockStorage/repertory-js
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

2.0.x fixes

Browse Source
This commit is contained in:
sgraves76
2025-02-13 11:55:03 -06:00
parent f0a6db2266
commit a4c7e699e1
12 changed files with 108 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.cspell/words.txt Normal file
View File

@@ -0,0 +1 @@
blockstorage

78
README.md
View File

@@ -1,6 +1,7 @@
# About
`repertory-js` is a Node.js module for interfacing with `repertory's` remote mount API.
`repertory-js` is a Node.js module for interfacing with `repertory's` remote
mount API.
## Installing
@@ -10,43 +11,44 @@ npm i @blockstorage/repertory-js
## Repertory Configuration
A Repertory mount must be active with the `EnableRemoteMount` setting enabled. `RemoteToken` should
also be set to a strong, random password.
A Repertory mount must be active with the `RemoteMount.Enable` setting enabled.
`RemoteMount.EncryptionToken` should also be set to a strong, random password.
### Enabling Sia Remote Mount API on Windows Systems
```shell
repertory.exe -unmount
repertory.exe -set RemoteMount.EnableRemoteMount true
repertory.exe -set RemoteMount.RemoteToken "my password"
repertory.exe -set RemoteMount.Enable true
repertory.exe -set RemoteMount.EncryptionToken 'my password'
[Optional - change listening port]
repertory.exe -set RemoteMount.RemotePort 20202
repertory.exe -set RemoteMount.ApiPort 20202
```
### Enabling Sia Remote Mount API on *NIX Systems
```shell
./repertory -unmount
./repertory -set RemoteMount.EnableRemoteMount true
./repertory -set RemoteMount.RemoteToken "my password"
./repertory -set RemoteMount.Enable true
./repertory -set RemoteMount.EncryptionToken 'my password'
[Optional - change listening port]
./repertory -set RemoteMount.RemotePort 20202
./repertory -set RemoteMount.ApiPort 20202
```
### Skynet and ScPrime Mounts
### S3 Mounts
* For Skynet mounts, add `-sk` argument to all commands listed above.
* For ScPrime mounts, add `-sp` argument to all commands listed above.
* For S3 mounts, add `-s3` and `--name '<my config name>'` argument to all
commands listed above.
## Module Environment Variables
* To successfully complete unit tests, a `repertory` mount supporting remote mount needs to be
active. Set the following environment variables prior to running tests:
* `TEST_HOST`
* `TEST_PASSWORD`
* `TEST_PORT`
* To successfully complete unit tests, a `repertory` mount supporting remote
mount needs to be active. Set the following environment variables prior to
running tests:
* `TEST_HOST`
* `TEST_PASSWORD`
* `TEST_PORT`
## Example API Usage
@@ -58,7 +60,7 @@ import * as rep from "@blockstorage/repertory-js";
// Repertory host settings
const MY_HOST_OR_IP = 'localhost';
const MY_PORT = 20000;
const MY_TOKEN = 'password';
const MY_PASSWORD = 'password';
// Progress callback for uploads / downloads
const progress_cb = (local_path, remote_path, progress, completed) => {
@@ -66,33 +68,29 @@ const progress_cb = (local_path, remote_path, progress, completed) => {
};
//************************************************************************************************//
// Step 1. Create a connection pool (recommended) //
//************************************************************************************************//
const conn = await rep.create_pool(8, MY_HOST_OR_IP, MY_PORT, MY_TOKEN);
/* Or create a single connection for light operations
//****************************************************************************//
// Step 1. Create a connection //
//****************************************************************************//
const conn = await rep.connect(MY_HOST_OR_IP, MY_PORT, MY_TOKEN);
*/
/* Disconnect when complete
await conn.disconnect();
*/
//************************************************************************************************//
// Step 2. Create an 'api' instance using the connection pool / connection //
//************************************************************************************************//
//****************************************************************************//
// Step 2. Create an 'api' instance using the connection //
//****************************************************************************//
const api = rep.create_api(conn);
//************************************************************************************************//
// Step 3. Use 'api' //
//************************************************************************************************//
//****************************************************************************//
// Step 3. Use 'api' //
//****************************************************************************//
//------------------------------------------------------------------------------------------------//
// *********** Directory Operations *********** //
//------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------//
// *********** Directory Operations *********** //
//----------------------------------------------------------------------------//
// Check if directory exists
const exists = await api.directory.exists('/my_directory');
@@ -125,9 +123,9 @@ await api.directory.create('/test');
await api.directory.remove('/test')
//------------------------------------------------------------------------------------------------//
// *********** File Operations *********** //
//------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------//
// *********** File Operations *********** //
//----------------------------------------------------------------------------//
// Check if file exists
const exists = await api.file.exists('/my_file.txt')
@@ -154,9 +152,9 @@ await api.file.upload('C:\\my_file.txt', '/my_file.txt', progress_cb, true);
await api.file.upload('C:\\my_file.txt', '/my_file.txt', progress_cb, false, true);
//------------------------------------------------------------------------------------------------//
// *********** Low-Level File Operations *********** //
//------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------//
// *********** Low-Level File Operations *********** //
//----------------------------------------------------------------------------//
// Create or open a remote file
{

16
cspell.json Normal file
View File

@@ -0,0 +1,16 @@
{
"$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json",
"version": "0.2",
"dictionaries": ["workspace-words", "user-words"],
"dictionaryDefinitions": [{
"name": "workspace-words",
"path": "./.cspell/words.txt",
"addWords": true
},
{
"name": "user-words",
"path": "C:\\.desktop\\.cspell\\user_words.txt",
"addWords": true
}
]
}

2
package.json
View File

@@ -42,8 +42,8 @@
},
"dependencies": {
"@stablelib/xchacha20poly1305": "^1.0.1",
"blake2": "^5.0.0",
"int64-buffer": "^1.0.0",
"socket-pool": "^1.2.3",
"text-encoding": "^0.7.0",
"uuid": "^8.3.2"
},

59
src/__tests__/connection_pool.test.js
View File

@@ -1,59 +0,0 @@
import connection_pool from '../networking/connection_pool';
import packet from '../networking/packet';
import connection from '../networking/connection';
jest.mock('../networking/connection');
test(`construction fails if pool size is <= 1`, () => {
expect(() => new connection_pool(1)).toThrow(Error);
expect(() => new connection_pool(0)).toThrow(Error);
expect(() => new connection_pool(-1)).toThrow(Error);
});
test(`error on socket release is ignored`, async () => {
const conn = new connection_pool(2, '', 20000);
let invoked = false;
jest.spyOn(conn.pool, 'acquire').mockImplementation(() => {
return {
release: () => {
invoked = true;
throw new Error('mock release error');
},
};
});
const mock_send = jest.fn();
connection.prototype.send = async () => {
return mock_send();
};
mock_send.mockResolvedValue(0);
expect(await conn.send('', new packet())).toEqual(0);
expect(invoked).toBeTruthy();
});
test(`connection pool send fails if socket acquire fails`, async () => {
const conn = new connection_pool(2, '', 20000);
jest.spyOn(conn.pool, 'acquire').mockImplementation(() => {
throw new Error('mock acquire exception');
});
await expect(conn.send('', new packet())).rejects.toThrow(Error);
});
test(`connection pool send fails when connection send fails`, async () => {
const conn = new connection_pool(2, '', 20000);
jest.spyOn(conn.pool, 'acquire').mockImplementation(() => {
return {
release: () => {},
};
});
const mock_send = jest.fn();
connection.prototype.send = async () => {
return mock_send();
};
mock_send.mockRejectedValue(new Error('mock send failed'));
await expect(conn.send('', new packet())).rejects.toThrow(Error);
});

2
src/__tests__/constants.test.js
View File

@@ -14,5 +14,5 @@ test(`'instance_id' is valid`, () => {
test(`'version' can be read from 'package.json'`, () => {
console.log(get_version());
expect(get_version()).toBe('1.4.0-r1');
expect(get_version()).toBe('2.0.0-r1');
});

165
src/__tests__/repertory.test.js
View File

@@ -1,18 +1,17 @@
import crypto from 'crypto';
import blake2 from 'blake2';
import fs from 'fs';
import { Uint64BE } from 'int64-buffer';
import * as repertory from '../index.js';
import connection from '../networking/connection';
import connection_pool from '../networking/connection_pool';
const TEST_HOST = process.env.TEST_HOST || 'localhost';
const TEST_PASSWORD = process.env.TEST_PASSWORD || '';
const TEST_PASSWORD = process.env.TEST_PASSWORD || 'cow_moose_doge_chicken';
const TEST_PORT = process.env.TEST_PORT || 20000;
const calculate_sha256 = (path) => {
return new Promise((resolve, reject) => {
const hash = crypto.createHash('sha256');
const hash = blake2.createHash('blake2b', { digestLength: 32 });
fs.createReadStream(path)
.on('data', (data) => hash.update(data))
@@ -35,56 +34,16 @@ const test_connection = (conn, should_be_connected) => {
};
test('can create a connection to repertory api', async () => {
console.log('TEST_PASSWORD', TEST_PASSWORD);
const conn = await repertory.connect(TEST_HOST, TEST_PORT, TEST_PASSWORD);
test_connection(conn, true);
await conn.disconnect();
});
test('create_pool returns a connection if pool size is <=1', async () => {
for (let i = 0; i < 2; i++) {
const conn = await repertory.create_pool(
i,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
expect(conn).toBeInstanceOf(connection);
test_connection(conn, true);
await conn.disconnect();
}
});
test('can create a connection pool', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
console.log(conn);
expect(conn).toBeInstanceOf(connection_pool);
expect(conn.host_or_ip).toEqual(TEST_HOST);
expect(conn.port).toEqual(TEST_PORT);
expect(conn.password).toEqual(TEST_PASSWORD);
expect(conn.shutdown).toEqual(false);
expect(conn.pool._pool.max).toEqual(2);
expect(conn.pool._pool.min).toEqual(2);
await conn.disconnect();
});
test('can get drive information using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
const di = await api.get_drive_information();
console.log(di);
expect(di.free).toBeDefined();
expect(di.total).toBeDefined();
@@ -94,12 +53,7 @@ test('can get drive information using api', async () => {
});
test('can create and remove a directory using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(await api.directory.create('/repertory_js')).toEqual(0);
expect(await api.directory.exists('/repertory_js')).toEqual(true);
@@ -110,12 +64,7 @@ test('can create and remove a directory using api', async () => {
});
test('can get directory list and snapshot using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
const test_results = async (remote_path, page_count, get_page) => {
@@ -127,10 +76,11 @@ test('can get directory list and snapshot using api', async () => {
console.log(items);
expect(items.length).toBeGreaterThanOrEqual(2);
expect(items[0].directory).toBeTruthy();
expect(items[0].path).toEqual('.');
expect(items[1].directory).toBeTruthy();
expect(items[1].path).toEqual('..');
expect(items[0].Directory).toBeTruthy();
expect(items[0].ApiPath).toEqual('.');
expect(items[1].Directory).toBeTruthy();
expect(items[1].ApiPath).toEqual('..');
}
};
@@ -153,12 +103,7 @@ test('can get directory list and snapshot using api', async () => {
});
test('can create, close and remove a file using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
const f = await api.file.create_or_open('/repertory_file.dat');
console.log(f);
@@ -175,12 +120,7 @@ test('can create, close and remove a file using api', async () => {
});
test('can open, close and remove a file using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
let f = await api.file.create_or_open('/repertory_file.dat');
expect(await f.close()).toEqual(0);
@@ -200,12 +140,7 @@ test('can open, close and remove a file using api', async () => {
});
test('can write to and read from a file using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
const f = await api.file.create_or_open('/repertory_file.dat');
@@ -227,12 +162,7 @@ test('can write to and read from a file using api', async () => {
});
test('can truncate a file using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
const f = await api.file.create_or_open('/repertory_file.dat');
@@ -253,12 +183,7 @@ test('can upload and download a file using api', async () => {
fs.unlinkSync('repertory_test.dat');
} catch {}
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(
await api.file.upload('test.dat', '/repertory_test.dat', (l, r, p, c) => {
@@ -289,12 +214,7 @@ test('can upload and download a file using api', async () => {
}, 60000);
test('can download and overwrite a file using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(
await api.file.upload('test.dat', '/repertory_test.dat', (l, r, p, c) => {
@@ -330,12 +250,7 @@ test('can download and overwrite a file using api', async () => {
}, 60000);
test('download fails if overwrite is false using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(
await api.file.upload('test.dat', '/repertory_test.dat', (l, r, p, c) => {
@@ -371,12 +286,7 @@ test('download fails if overwrite is false using api', async () => {
}, 60000);
test('can upload and overwrite a file using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(
await api.file.upload('test.dat', '/repertory_test.dat', (l, r, p, c) => {
@@ -401,12 +311,7 @@ test('can upload and overwrite a file using api', async () => {
}, 60000);
test('upload fails if overwrite is false using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(
await api.file.upload('test.dat', '/repertory_test.dat', (l, r, p, c) => {
@@ -431,12 +336,7 @@ test('upload fails if overwrite is false using api', async () => {
}, 60000);
test('can resume download using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(
await api.file.upload('test.dat', '/repertory_test.dat', (l, r, p, c) => {
@@ -474,12 +374,7 @@ test('can resume download using api', async () => {
}, 60000);
test('can resume upload using api', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
const fd = fs.openSync('test.dat', 'r');
@@ -524,24 +419,14 @@ test('can resume upload using api', async () => {
}, 60000);
test('exists returns false if directory is not found', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(await api.directory.exists('/cow')).toEqual(false);
await conn.disconnect();
});
test('exists returns false if file is not found', async () => {
const conn = await repertory.create_pool(
2,
TEST_HOST,
TEST_PORT,
TEST_PASSWORD
);
const conn = await repertory.create(TEST_HOST, TEST_PORT, TEST_PASSWORD);
const api = repertory.create_api(conn);
expect(await api.file.exists('/cow')).toEqual(false);
await conn.disconnect();

17
src/index.js
View File

@@ -1,6 +1,5 @@
import file from './io/file';
import connection from './networking/connection';
import connection_pool from './networking/connection_pool';
import * as ops from './ops';
export * as byte_order from './utils/byte_order';
@@ -9,7 +8,13 @@ export { default as packet } from './networking/packet';
export const connect = async (host_or_ip, port, password) => {
const conn = new connection(host_or_ip, port, password);
await conn.connect();
try {
await conn.connect();
} catch (e) {
await conn.disconnect();
throw e;
}
return conn;
};
@@ -85,10 +90,6 @@ export const create_api = (conn) => {
};
};
export const create_pool = async (pool_size, host_or_ip, port, password) => {
if (pool_size <= 1) {
return connect(host_or_ip, port, password);
}
return new connection_pool(pool_size, host_or_ip, port, password);
export const create = async (host_or_ip, port, password) => {
return connect(host_or_ip, port, password);
};

19
src/networking/connection.js
View File

@@ -39,21 +39,21 @@ export default class connection {
this.host_or_ip,
(err) => {
if (err) {
console.log(err);
console.error(err);
return reject(err);
}
this.reject = reject;
this.resolve = resolve;
this.connected = true;
this.setup_socket();
}
);
});
} catch (err) {
return Promise.reject(new Error(`'connect()' failed: ${err}`));
}
this.connected = true;
this.setup_socket();
}
}
@@ -88,6 +88,7 @@ export default class connection {
};
};
this.buffer = null;
const response = new packet(this.password);
response.buffer = new Uint8Array(packet_data);
response
@@ -100,7 +101,7 @@ export default class connection {
}
})
.catch((e) => {
console.log(e);
console.error(e);
const { reject } = complete();
if (reject) {
reject(e);
@@ -118,7 +119,7 @@ export default class connection {
cleanup();
this.connected = false;
console.log(e);
console.error(e);
if (reject) {
reject(e);
}
@@ -132,7 +133,7 @@ export default class connection {
cleanup();
this.connected = false;
console.log('socket closed');
console.warn('socket closed');
if (reject) {
reject(new Error('socket closed'));
}
@@ -149,7 +150,7 @@ export default class connection {
this.connected = false;
}
} catch (e) {
console.log(e);
console.error(e);
}
}
@@ -160,7 +161,7 @@ export default class connection {
packet.encode_top_utf8(constants.instance_id);
packet.encode_top_ui32(0); // Service flags
packet.encode_top_utf8(constants.get_version());
packet.encode_top_utf8(nonce);
packet.encode_top_utf8(this.nonce);
await packet.encrypt();
packet.encode_top_ui32(packet.buffer.length);
return new Promise((resolve, reject) => {

69
src/networking/connection_pool.js
View File

@@ -1,69 +0,0 @@
import Pool from 'socket-pool';
import connection from './connection';
export default class connection_pool {
constructor(pool_size, host_or_ip, port, password) {
this.host_or_ip = host_or_ip;
this.port = port;
this.password = password;
if (pool_size > 1) {
this.pool = new Pool({
connect: { host: host_or_ip, port: port },
connectTimeout: 5000,
pool: { max: pool_size, min: 2 },
});
} else {
throw new Error("'pool_size' must be > 1");
}
}
host_or_ip = '';
next_thread_id = 1;
password = '';
port = 20000;
pool;
shutdown = false;
async disconnect() {
await this.pool._pool.drain();
await this.pool._pool.clear();
this.pool = null;
this.shutdown = true;
}
async send(method_name, packet, optional_thread_id) {
try {
const socket = await this.pool.acquire();
if (!socket.thread_id) {
socket.thread_id = this.next_thread_id++;
}
const cleanup = () => {
try {
socket.release();
} catch (err) {
console.log(`'release()' failed: ${err}`);
}
};
try {
const result = await new connection(
this.host_or_ip,
this.port,
this.password,
socket
).send(method_name, packet, optional_thread_id || socket.thread_id);
cleanup();
return result;
} catch (err) {
cleanup();
return Promise.reject(
new Error(`'send(${method_name})' failed: ${err}`)
);
}
} catch (err) {
return Promise.reject(new Error(`'acquire()' socket failed: ${err}`));
}
}
}

6
src/networking/packet.js
View File

@@ -1,6 +1,6 @@
import { randomBytes } from 'crypto';
import { Int64BE, Uint64BE } from 'int64-buffer';
import crypto from 'crypto';
import blake2 from 'blake2';
import { TextEncoder } from 'text-encoding';
import { getCustomEncryption } from '../utils/constants';
import {
@@ -156,7 +156,7 @@ export default class packet {
decrypt = async () => {
try {
let hash = crypto.createHash('sha256');
let hash = blake2.createHash('blake2b', { digestLength: 32 });
hash = hash.update(new TextEncoder().encode(this.token));
const key = Uint8Array.from(hash.digest());
@@ -282,7 +282,7 @@ export default class packet {
encrypt = async (nonce) => {
try {
let hash = crypto.createHash('sha256');
let hash = blake2.createHash('blake2b', { digestLength: 32 });
hash = hash.update(new TextEncoder().encode(this.token));
const key = Uint8Array.from(hash.digest());

8
src/ops/index.js
View File

@@ -167,14 +167,14 @@ export const download_file = async (
try {
await src.close();
} catch (err) {
console.log(err);
console.error(err);
}
try {
if (fd !== undefined) {
fs.closeSync(fd);
}
} catch (err) {
console.log(err);
console.error(err);
}
};
@@ -466,14 +466,14 @@ export const upload_file = async (
try {
fs.closeSync(src_fd);
} catch (err) {
console.log(err);
console.error(err);
}
try {
if (f) {
await f.close();
}
} catch (err) {
console.log(err);
console.error(err);
}
};
try {