describe('Split sentence', () => { test('Should split a sentence into tokens', () => { const input = "This isn't tokenized, maybe."; const tokenizer = new BertWordPieceTokenizer({ vocabContent: vocabEn }); const actual = tokenizer.splitSentence(input); expect(actual.length).toEqual(11); expect(actual[0].token).toEqual('This'); expect(actual[6].start).toEqual(11); expect(actual[6].end).toEqual(19); expect(actual[6].type).toEqual('word'); }); });
it('with range option as array composed of single object', () => { const content = `hello world, on ${Math.round(Math.random() * 100000)}`; const range = [{ start: 3, end: 12 }]; return createRandomDomain(content, '/streaming.mp4', null, app) .then(({ domain }) => unregisteredApp.webFetch(`safe://${domain}/streaming.mp4`, { range }) .then((data) => { should.not.exist(data.parts); should(data.body.toString()).equal(content.substring(3, 13)); should(data.headers['Content-Range']).equal(`bytes 3-12/${content.length}`); return should(data.headers['Content-Length']).equal((range[0].end - range[0].start) + 1); })); });
for (var j = 0, jj = ranges[i].end - ranges[i].start + 1; j < jj; j++) { ranges[i].ids[j] = getUshort(data, p + idOffset); idOffset += 2;
var i = 0; while (i < horizon.length && horizon[i].end <= boundary.y1) { i++; changedHorizon[changedHorizon.length - 1].end = horizonPart.end; } else { changedHorizon.push({ start: horizonPart.start, end: horizonPart.end, boundary: useBoundary }); if (boundary.y2 < horizon[j].end) { changedHorizon[changedHorizon.length - 1].end = boundary.y2; changedHorizon.push({ start: boundary.y2, end: horizon[j].end, boundary: horizon[j].boundary }); for (k = j + 1; !used && k < horizon.length && horizon[k].end <= affectedBoundary.y2; k++) { used = horizon[k].boundary === affectedBoundary;
bounds.forEach(function (boundary) { var i = 0; while (i < horizon.length && horizon[i].end <= boundary.y1) { i++; var useBoundary = affectedBoundary.x2 > boundary.x2 ? affectedBoundary : boundary; if (lastBoundary === useBoundary) { changedHorizon[changedHorizon.length - 1].end = horizonPart.end; } else { changedHorizon.push({ start: horizonPart.start, end: horizonPart.end, boundary: useBoundary }); }); if (boundary.y2 < horizon[j].end) { changedHorizon[changedHorizon.length - 1].end = boundary.y2; changedHorizon.push({ start: boundary.y2, end: horizon[j].end, boundary: horizon[j].boundary }); for (k = j + 1; !used && k < horizon.length && horizon[k].end <= affectedBoundary.y2; k++) { used = horizon[k].boundary === affectedBoundary;