How to use the tokenize function from esprima

preProcessData.js

GitHub: suryaumapathy2812/tensor_model

5
6
7
8
9
10
11
12
13
14
15
const rawData = fs.readFileSync('./src/dataset.json');
const dataset = JSON.parse(rawData);


function tokenizeCode(code) {
    try {
        const tokens = esprima.tokenize(code);
        return tokens.map((token) => token.value);
    } catch (error) {
        console.error(error);
        return [];

0

0

0