Utilities / extract-mongo-schema

Download as
Browse Code ยป

Commit 8fe980dc321cc1537e316fcb17e87764dbfe1178

Authored by Tai Tran
0 parents
Exists in master

support mongo3 & clone from https://github.com/perak/extract-mongo-schema

Showing 8 changed files with 1078 additions and 0 deletions Side-by-side Diff

... ... @@ -0,0 +1 @@
  1 +node_modules/
... ... @@ -0,0 +1,312 @@
  1 +# Extract Mongo Schema
  2 +
  3 +Extract (and visualize) schema from Mongo database, including foreign keys. Output is simple json file or html with dagre/d3.js diagram (depending on command line options).
  4 +
  5 +## Installation
  6 +
  7 +```sh
  8 +git clone <gitlab>
  9 +npm install
  10 +```
  11 +
  12 +## Usage
  13 +
  14 +```sh
  15 +
  16 +Usage:
  17 + node cli.js -d connection_string -o schema.json -f json
  18 + -d, --database Database connection string. Example: "mongodb://localhost:3001/meteor".
  19 + -o, --output Output file
  20 + -f, --format Output file format. Can be "json" or "html-diagram". Default is "json".
  21 + -c, --collection Comma separated list of collections to analyze. Example: "collection1,collection2".
  22 + -a, --array Comma separated list of types of arrays to analyze. Example: "Uint8Array,ArrayBuffer,Array".
  23 + -r, --raw Shows the exact list of types with frequency instead of the most frequent type only.
  24 + -l, --limit Number of records to parse to get the schema, default is 100.
  25 + -n, --dont-follow-fk Don't follow specified foreign key. Can be simply "fieldName" (all collections) or "collectionName:fieldName" (only for given collection).
  26 +
  27 +```
  28 +
  29 +
  30 +## Example usage
  31 +
  32 +**Extract schema into json**
  33 +
  34 +```
  35 +node cli.js -d "mongodb://localhost:3001/meteor" -o schema.json
  36 +```
  37 +
  38 +
  39 +**Extract schema into html**
  40 +
  41 +```
  42 +node cli.js -d "mongodb://localhost:3001/meteor" -o schema.html -f html-diagram
  43 +```
  44 +
  45 +**Extract specific collections in raw format and analyze Array items**
  46 +
  47 +```
  48 +node cli.js -d "mongodb://localhost:3001/meteor" -o schema.json -c "collection1,collection2,collection3" -a "Array" -r
  49 +```
  50 +
  51 +Open html in your browser and you'll see rendered ER diagram.
  52 +
  53 +
  54 +**Ignore some foreign keys**
  55 +
  56 +Use `-n` switch to prevent detecting and drawing links for specified fields. You can specify simply `fieldName` (that applies to all collections) or `collectionName:fieldName` (foreign key is ignored only in given collection).
  57 +
  58 +Example:
  59 +
  60 +```
  61 +node cli.js -d "mongodb://localhost:3001/meteor" -o schema.html -f html-diagram -n createdBy -n users:modifiedBy
  62 +```
  63 +*(in this example: any foreign key named "createdBy" will be ignored. Also "modifiedBy" but only in users collection)*
  64 +
  65 +
  66 +## Example output .html (screenshot)
  67 +
  68 +![Alt text](/preview.png?raw=true "Preview")
  69 +
  70 +
  71 +## Example output .json
  72 +
  73 +**schema.json**
  74 +
  75 +```json
  76 +{
  77 + "customers": {
  78 + "_id": {
  79 + "primaryKey": true,
  80 + "type": "string",
  81 + "required": true
  82 + },
  83 + "name": {
  84 + "type": "string",
  85 + "required": true
  86 + },
  87 + "phone": {
  88 + "type": "string",
  89 + "required": true
  90 + },
  91 + "email": {
  92 + "type": "string",
  93 + "required": true
  94 + },
  95 + "note": {
  96 + "type": "string",
  97 + "required": true
  98 + },
  99 + "createdAt": {
  100 + "type": "Date",
  101 + "required": true
  102 + },
  103 + "createdBy": {
  104 + "key": true,
  105 + "type": "string",
  106 + "required": true
  107 + },
  108 + "modifiedAt": {
  109 + "type": "Date",
  110 + "required": true
  111 + },
  112 + "modifiedBy": {
  113 + "key": true,
  114 + "type": "string",
  115 + "required": true
  116 + },
  117 + "ownerId": {
  118 + "key": true,
  119 + "type": "string",
  120 + "required": true
  121 + }
  122 + },
  123 + "invoices": {
  124 + "_id": {
  125 + "primaryKey": true,
  126 + "type": "string",
  127 + "required": true
  128 + },
  129 + "invoiceNumber": {
  130 + "type": "string",
  131 + "required": true
  132 + },
  133 + "date": {
  134 + "type": "Date",
  135 + "required": true
  136 + },
  137 + "customerId": {
  138 + "foreignKey": true,
  139 + "references": "customers",
  140 + "key": true,
  141 + "type": "string",
  142 + "required": true
  143 + },
  144 + "createdAt": {
  145 + "type": "Date",
  146 + "required": true
  147 + },
  148 + "createdBy": {
  149 + "key": true,
  150 + "type": "string",
  151 + "required": true
  152 + },
  153 + "modifiedAt": {
  154 + "type": "Date",
  155 + "required": true
  156 + },
  157 + "modifiedBy": {
  158 + "key": true,
  159 + "type": "string",
  160 + "required": true
  161 + },
  162 + "ownerId": {
  163 + "key": true,
  164 + "type": "string",
  165 + "required": true
  166 + },
  167 + "totalAmount": {
  168 + "type": "number",
  169 + "required": true
  170 + }
  171 + },
  172 + "users": {
  173 + "_id": {
  174 + "primaryKey": true,
  175 + "type": "string",
  176 + "required": true
  177 + },
  178 + "createdAt": {
  179 + "type": "Date",
  180 + "required": true
  181 + },
  182 + "services": {
  183 + "type": "Object",
  184 + "structure": {
  185 + "password": {
  186 + "type": "Object",
  187 + "structure": {
  188 + "bcrypt": {
  189 + "type": "string",
  190 + "required": true
  191 + }
  192 + },
  193 + "required": true
  194 + },
  195 + "resume": {
  196 + "type": "Object",
  197 + "structure": {
  198 + "loginTokens": {
  199 + "type": "Array",
  200 + "required": true
  201 + }
  202 + },
  203 + "required": true
  204 + }
  205 + },
  206 + "required": true
  207 + },
  208 + "emails": {
  209 + "type": "Array",
  210 + "required": true
  211 + },
  212 + "roles": {
  213 + "type": "Array",
  214 + "required": true
  215 + },
  216 + "profile": {
  217 + "type": "Object",
  218 + "structure": {
  219 + "name": {
  220 + "type": "string",
  221 + "required": true
  222 + },
  223 + "email": {
  224 + "type": "string",
  225 + "required": true
  226 + },
  227 + "facebook": {
  228 + "type": "string",
  229 + "required": true
  230 + },
  231 + "google": {
  232 + "type": "string",
  233 + "required": true
  234 + },
  235 + "twitter": {
  236 + "type": "string",
  237 + "required": true
  238 + },
  239 + "website": {
  240 + "type": "string",
  241 + "required": true
  242 + }
  243 + },
  244 + "required": true
  245 + }
  246 + },
  247 + "meteor_accounts_loginServiceConfiguration": {},
  248 + "invoice_items": {
  249 + "_id": {
  250 + "primaryKey": true,
  251 + "type": "string",
  252 + "required": true
  253 + },
  254 + "description": {
  255 + "type": "string",
  256 + "required": true
  257 + },
  258 + "quantity": {
  259 + "type": "number",
  260 + "required": true
  261 + },
  262 + "price": {
  263 + "type": "number",
  264 + "required": true
  265 + },
  266 + "invoiceId": {
  267 + "key": true,
  268 + "foreignKey": true,
  269 + "references": "invoices",
  270 + "type": "string",
  271 + "required": true
  272 + },
  273 + "createdAt": {
  274 + "type": "Date",
  275 + "required": true
  276 + },
  277 + "createdBy": {
  278 + "key": true,
  279 + "foreignKey": true,
  280 + "references": "users",
  281 + "type": "string",
  282 + "required": true
  283 + },
  284 + "modifiedAt": {
  285 + "type": "Date",
  286 + "required": true
  287 + },
  288 + "modifiedBy": {
  289 + "key": true,
  290 + "foreignKey": true,
  291 + "references": "users",
  292 + "type": "string",
  293 + "required": true
  294 + },
  295 + "ownerId": {
  296 + "key": true,
  297 + "foreignKey": true,
  298 + "references": "users",
  299 + "type": "string",
  300 + "required": true
  301 + },
  302 + "amount": {
  303 + "type": "number",
  304 + "required": true
  305 + }
  306 + }
  307 +}
  308 +```
  309 +
  310 +
  311 +That's all folks.
  312 +Enjoy! :)
... ... @@ -0,0 +1,150 @@
  1 +#! /usr/bin/env node
  2 +
  3 +const commandLineArgs = require("command-line-args");
  4 +const fs = require("fs");
  5 +const path = require("path");
  6 +const replaceExt = require("replace-ext");
  7 +const extractMongoSchema = require("./extract-mongo-schema");
  8 +
  9 +const optionDefinitions = [
  10 + { name: "database", alias: "d", type: String },
  11 + { name: "output", alias: "o", type: String },
  12 + { name: "format", alias: "f", type: String },
  13 + { name: "collection", alias: "c", type: String },
  14 + { name: "array", alias: "a", type: String },
  15 + { name: "raw", alias: "r", type: Boolean, defaultValue: false },
  16 + { name: "limit", alias: "l", type: Number, defaultValue: 100 },
  17 + { name: "dont-follow-fk", alias: "n", multiple: true, type: String }
  18 +];
  19 +
  20 +const args = commandLineArgs(optionDefinitions);
  21 +
  22 +console.log("");
  23 +console.log("Extract schema from Mongo database (including foreign keys)");
  24 +
  25 +var printUsage = function() {
  26 + console.log("");
  27 + console.log("Usage:");
  28 + console.log("\textract-mongo-schema -d connection_string -o schema.json");
  29 + console.log("\t\t-d, --database string\tDatabase connection string. Example: \"mongodb://localhost:3001/meteor\".");
  30 + console.log("\t\t-o, --output string\tOutput file");
  31 + console.log("\t\t-f, --format string\tOutput file format. Can be \"json\" or \"html-diagram\".");
  32 + console.log("\t\t-c, --collection\tComma separated list of collections to analyze. Example: \"collection1,collection2\".");
  33 + console.log("\t\t-a, --array\tComma separated list of types of arrays to analyze. Example: \"Uint8Array,ArrayBuffer,Array\".");
  34 + console.log("\t\t-r, --raw\tShows the exact list of types with frequency instead of the most frequent type only.");
  35 + console.log("\t\t-l, --limit\tChanges the amount of items to parse from the collections. Default is 100.");
  36 + console.log("\t\t-n, --dont-follow-fk string\tDon't follow specified foreign key. Can be simply \"fieldName\" (all collections) or \"collectionName:fieldName\" (only for given collection).");
  37 + console.log("");
  38 + console.log("Enjoy! :)");
  39 + console.log("");
  40 +};
  41 +
  42 +if(!args.database) {
  43 + console.log("");
  44 + console.log("Database connection string is missing.");
  45 + printUsage();
  46 + process.exit(1);
  47 +}
  48 +
  49 +if(!args.output) {
  50 + console.log("");
  51 + console.log("Output path is missing.");
  52 + printUsage();
  53 + process.exit(1);
  54 +}
  55 +
  56 +if(fs.existsSync(args.output)) {
  57 + var outputStat = fs.lstatSync(args.output);
  58 +
  59 + if(outputStat.isDirectory()) {
  60 + console.log("Error: output \"" + args.output + "\" is not a file.");
  61 + process.exit(1);
  62 + }
  63 +}
  64 +
  65 +var collectionList = null;
  66 +if(args.collection) {
  67 + collectionList = args.collection.split(",");
  68 +}
  69 +
  70 +var arrayList = null;
  71 +if(args.array) {
  72 + arrayList = args.array.split(",");
  73 +}
  74 +
  75 +var outputFormat = args.format || "json";
  76 +
  77 +var dontFollowTMP = args["dont-follow-fk"] || [];
  78 +
  79 +var dontFollowFK = {
  80 + __ANY__: {}
  81 +};
  82 +
  83 +dontFollowTMP.map(function(df) {
  84 + var dfArray = df.split(":");
  85 +
  86 + var collection = "";
  87 + var field = "";
  88 +
  89 + if(dfArray.length > 1) {
  90 + collection = dfArray[0];
  91 + field = dfArray[1];
  92 + } else {
  93 + collection = "__ANY__";
  94 + field = dfArray[0];
  95 + }
  96 + dontFollowFK[collection][field] = true;
  97 +});
  98 +
  99 +console.log("");
  100 +console.log("Extracting...");
  101 +
  102 +var opts = {
  103 + collectionList: collectionList,
  104 + arrayList:arrayList,
  105 + raw: args.raw,
  106 + limit: args.limit,
  107 + dontFollowFK: dontFollowFK
  108 +};
  109 +
  110 +extractMongoSchema.extractMongoSchema(args.database, opts, function(err, schema) {
  111 + if(err) {
  112 + console.log(err);
  113 + process.exit(1);
  114 + }
  115 +
  116 + if(outputFormat === "json") {
  117 + try {
  118 + fs.writeFileSync(args.output, JSON.stringify(schema, null, "\t"), "utf8");
  119 + } catch(e) {
  120 + console.log("Error: cannot write output \"" + args.output + "\". " + e.message);
  121 + process.exit(1);
  122 + }
  123 + }
  124 +
  125 + if(outputFormat === "html-diagram") {
  126 + var templateFileName = path.join(__dirname, "/template-html-diagram.html");
  127 +
  128 + // read input file
  129 + var templateHTML = "";
  130 + try {
  131 + templateHTML = fs.readFileSync(templateFileName, "utf8");
  132 + } catch(e) {
  133 + console.log("Error: cannot read template file \"" + templateFileName + "\". " + e.message);
  134 + process.exit(1);
  135 + }
  136 +
  137 + templateHTML = templateHTML.replace("{/*DATA_HERE*/}", JSON.stringify(schema, null, "\t"));
  138 +
  139 + try {
  140 + fs.writeFileSync(args.output, templateHTML, "utf8");
  141 + } catch(e) {
  142 + console.log("Error: cannot write output \"" + args.output + "\". " + e.message);
  143 + process.exit(1);
  144 + }
  145 + }
  146 +
  147 +
  148 + console.log("Success.");
  149 + console.log("");
  150 +});
extract-mongo-schema.js
Diff comments   View file @ 8fe980d
... ... @@ -0,0 +1,220 @@
  1 +var MongoClient = require("mongodb").MongoClient;
  2 +var wait = require("wait.for");
  3 +
  4 +var getSchema = function(url, opts) {
  5 + var client = wait.forMethod(MongoClient, "connect", url, { useNewUrlParser: true, useUnifiedTopology: true });
  6 +
  7 + var db = client.db("speedoc");
  8 + var l = wait.forMethod(db, "collections");
  9 +
  10 + var collectionInfos = l;//wait.forMethod(l, "toArray");
  11 + var schema = {};
  12 + var collections = {};
  13 +
  14 + var findRelatedCollection = function(value, field) {
  15 + for(var collectionName in collections) {
  16 + var related = wait.forMethod(collections[collectionName].collection, "findOne", { _id: value });
  17 + if(related) {
  18 + delete field["key"];
  19 + field["foreignKey"] = true;
  20 + field["references"] = collectionName;
  21 + } else {
  22 + field["key"] = true;
  23 + }
  24 + }
  25 + };
  26 +
  27 + var setTypeName = function(item) {
  28 + var typeName = typeof item;
  29 + if(typeName === "object") {
  30 + typeName = Object.prototype.toString.call(item);
  31 + }
  32 + typeName = typeName.replace("[object ", "");
  33 + typeName = typeName.replace("]", "");
  34 + return typeName;
  35 + };
  36 +
  37 + var getDocSchema = function(collectionName, doc, docSchema) {
  38 + for(var key in doc) {
  39 + if(!docSchema[key]) {
  40 + docSchema[key] = { "types": {} };
  41 + }
  42 +
  43 + if(!docSchema[key]["types"]) {
  44 + docSchema[key]["types"] = {};
  45 + }
  46 + var typeName = setTypeName(doc[key]);
  47 +
  48 + if(!docSchema[key]["types"][typeName]) {
  49 + docSchema[key]["types"][typeName] = { frequency: 0 };
  50 + }
  51 + docSchema[key]["types"][typeName]["frequency"]++;
  52 +
  53 + if(typeName == "string" && /^[23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz]{17}$/.test(doc[key])) {
  54 + if(key == "_id") {
  55 + docSchema[key]["primaryKey"] = true;
  56 + } else {
  57 + // only if is not already processes
  58 + if(!docSchema[key]["foreignKey"] || !docSchema[key]["references"]) {
  59 + // only if is not ignored
  60 + if(!(opts.dontFollowFK["__ANY__"][key] || (opts.dontFollowFK[collectionName] && opts.dontFollowFK[collectionName][key]))) {
  61 + findRelatedCollection(doc[key], docSchema[key]);
  62 + }
  63 + }
  64 + }
  65 + }
  66 +
  67 + if(typeName == "Object") {
  68 + if(!docSchema[key]["types"][typeName]["structure"]) {
  69 + docSchema[key]["types"][typeName]["structure"] = {};
  70 + }
  71 + getDocSchema(collectionName, doc[key], docSchema[key]["types"][typeName]["structure"]);
  72 + }
  73 +
  74 + if(opts.arrayList && opts.arrayList.indexOf(typeName) !== -1) {
  75 + if(!docSchema[key]["types"][typeName]["structure"]) {
  76 + docSchema[key]["types"][typeName]["structure"] = { "types": {} };
  77 + }
  78 +
  79 + if(!docSchema[key]["types"][typeName]["structure"]["types"]) {
  80 + docSchema[key]["types"][typeName]["structure"]["types"] = {};
  81 + }
  82 + for(var i = 0; i < doc[key].length; i++) {
  83 + var typeNameArray = setTypeName(doc[key][i]);
  84 + if(typeNameArray === "Object") {
  85 + if(!docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray]) {
  86 + docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray] = { "structure": {} };
  87 + }
  88 +
  89 + if(!docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray]["structure"]) {
  90 + docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray]["structure"] = {};
  91 + }
  92 + getDocSchema(collectionName, doc[key][i], docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray]["structure"]);
  93 + } else {
  94 + if(!docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray]) {
  95 + docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray] = { frequency: 0 };
  96 + }
  97 + docSchema[key]["types"][typeName]["structure"]["types"][typeNameArray]["frequency"]++;
  98 + }
  99 + }
  100 + }
  101 + }
  102 + };
  103 +
  104 + var setMostFrequentType = function(field, processed) {
  105 + var max = 0;
  106 + var notNull = true;
  107 + for(var typeName in field["types"]) {
  108 + if(typeName == "Null") {
  109 + notNull = false;
  110 + }
  111 + field["types"][typeName]["frequency"] = field["types"][typeName]["frequency"] / processed;
  112 + if(field["types"][typeName]["frequency"] > max) {
  113 + max = field["types"][typeName]["frequency"];
  114 + if(typeName != "undefined" && typeName != "Null") {
  115 + field["type"] = typeName;
  116 + }
  117 + }
  118 + }
  119 + return notNull;
  120 + }
  121 +
  122 + var mostFrequentType = function(docSchema, processed) {
  123 + if(processed) {
  124 + for(var fieldName in docSchema) {
  125 + if(docSchema[fieldName]) {
  126 + var notNull = setMostFrequentType(docSchema[fieldName], processed);
  127 + if(!docSchema[fieldName]["type"]) {
  128 + docSchema[fieldName]["type"] = "undefined";
  129 + notNull = false;
  130 + }
  131 +
  132 + var dataType = docSchema[fieldName]["type"];
  133 + if(dataType == "Object") {
  134 + mostFrequentType(docSchema[fieldName]["types"][dataType]["structure"], processed);
  135 + docSchema[fieldName]["structure"] = docSchema[fieldName]["types"][dataType]["structure"];
  136 + }
  137 +
  138 + if(opts.arrayList && opts.arrayList.indexOf(dataType) !== -1) {
  139 + if(Object.keys(docSchema[fieldName]["types"][dataType]["structure"]["types"])[0] == "Object") {
  140 + mostFrequentType(docSchema[fieldName]["types"][dataType]["structure"]["types"]["Object"]["structure"], processed);
  141 + docSchema[fieldName]["types"][dataType]["structure"]["type"] = "Object";
  142 + docSchema[fieldName]["types"][dataType]["structure"]["structure"] = docSchema[fieldName]["types"][dataType]["structure"]["types"]["Object"]["structure"];
  143 + delete docSchema[fieldName]["types"][dataType]["structure"]["types"];
  144 + } else {
  145 + mostFrequentType(docSchema[fieldName]["types"][dataType], processed);
  146 + }
  147 + docSchema[fieldName]["structure"] = docSchema[fieldName]["types"][dataType]["structure"];
  148 + }
  149 +
  150 + delete docSchema[fieldName]["types"];
  151 +
  152 + docSchema[fieldName]["required"] = notNull;
  153 + }
  154 + }
  155 + }
  156 + };
  157 +
  158 + if(opts.collectionList != null) {
  159 + for(var i = collectionInfos.length - 1; i >= 0; i--) {
  160 + if(opts.collectionList.indexOf(collectionInfos[i].collectionName) == -1) {
  161 + collectionInfos.splice(i, 1);
  162 + }
  163 + }
  164 + }
  165 +
  166 + collectionInfos.map(function(collectionInfo, index) {
  167 + var collectionData = {};
  168 + collections[collectionInfo.collectionName] = collectionData;
  169 + collectionData["collection"] = db.collection(collectionInfo.collectionName);
  170 + });
  171 +
  172 + collectionInfos.map(function(collectionInfo, index) {
  173 + collectionData = collections[collectionInfo.collectionName];
  174 + var docSchema = {};
  175 + schema[collectionInfo.collectionName] = docSchema;
  176 + var cur = wait.forMethod(collectionData["collection"], "find", {}, { limit: opts.limit });
  177 + var docs = wait.forMethod(cur, "toArray");
  178 + docs.map(function(doc) {
  179 + getDocSchema(collectionInfo.collectionName, doc, docSchema);
  180 + });
  181 + if(!opts.raw) {
  182 + mostFrequentType(docSchema, docs.length);
  183 + }
  184 + });
  185 +
  186 + // db.close();
  187 + return schema;
  188 +};
  189 +
  190 +
  191 +var printSchema = function(url, opts, cb) {
  192 + var schema = null;
  193 + try {
  194 + var schema = getSchema(url, opts);
  195 + } catch(err) {
  196 + if(cb) {
  197 + cb(err, null);
  198 + } else {
  199 + console.log(err);
  200 + }
  201 + return;
  202 + }
  203 +
  204 + if(cb) {
  205 + cb(null, schema);
  206 + }
  207 +
  208 + return schema;
  209 +};
  210 +
  211 +var extractMongoSchema = function(url, opts, cb) {
  212 + wait.launchFiber(printSchema, url, opts, cb);
  213 +};
  214 +
  215 +
  216 +if(typeof module != "undefined" && module.exports) {
  217 + module.exports.extractMongoSchema = extractMongoSchema;
  218 +} else {
  219 + this.extractMongoSchema = extractMongoSchema;
  220 +}
... ... @@ -0,0 +1,157 @@
  1 +{
  2 + "name": "extract-mongo-schema",
  3 + "version": "0.2.9",
  4 + "lockfileVersion": 1,
  5 + "requires": true,
  6 + "dependencies": {
  7 + "bson": {
  8 + "version": "1.1.1",
  9 + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.1.tgz",
  10 + "integrity": "sha512-jCGVYLoYMHDkOsbwJZBCqwMHyH4c+wzgI9hG7Z6SZJRXWr+x58pdIbm2i9a/jFGCkRJqRUr8eoI7lDWa0hTkxg=="
  11 + },
  12 + "command-line-args": {
  13 + "version": "3.0.5",
  14 + "resolved": "http://registry.npmjs.org/command-line-args/-/command-line-args-3.0.5.tgz",
  15 + "integrity": "sha1-W9StReeYPlwTRJGOQCgO4mk8WsA=",
  16 + "requires": {
  17 + "array-back": "^1.0.4",
  18 + "feature-detect-es6": "^1.3.1",
  19 + "find-replace": "^1.0.2",
  20 + "typical": "^2.6.0"
  21 + },
  22 + "dependencies": {
  23 + "array-back": {
  24 + "version": "1.0.4",
  25 + "resolved": "http://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz",
  26 + "integrity": "sha1-ZEun8JX3/898Q7Xw3DnTwfA8Bjs=",
  27 + "requires": {
  28 + "typical": "^2.6.0"
  29 + }
  30 + },
  31 + "feature-detect-es6": {
  32 + "version": "1.3.1",
  33 + "resolved": "http://registry.npmjs.org/feature-detect-es6/-/feature-detect-es6-1.3.1.tgz",
  34 + "integrity": "sha1-+IhzavnLDJH1VmO/pHYuuW7nBH8=",
  35 + "requires": {
  36 + "array-back": "^1.0.3"
  37 + }
  38 + },
  39 + "find-replace": {
  40 + "version": "1.0.3",
  41 + "resolved": "http://registry.npmjs.org/find-replace/-/find-replace-1.0.3.tgz",
  42 + "integrity": "sha1-uI5zZNLZyVlVnziMZmcNYTBEH6A=",
  43 + "requires": {
  44 + "array-back": "^1.0.4",
  45 + "test-value": "^2.1.0"
  46 + },
  47 + "dependencies": {
  48 + "test-value": {
  49 + "version": "2.1.0",
  50 + "resolved": "http://registry.npmjs.org/test-value/-/test-value-2.1.0.tgz",
  51 + "integrity": "sha1-Edpv9nDzRxpztiXKTz/c97t0gpE=",
  52 + "requires": {
  53 + "array-back": "^1.0.3",
  54 + "typical": "^2.6.0"
  55 + }
  56 + }
  57 + }
  58 + },
  59 + "typical": {
  60 + "version": "2.6.1",
  61 + "resolved": "http://registry.npmjs.org/typical/-/typical-2.6.1.tgz",
  62 + "integrity": "sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0="
  63 + }
  64 + }
  65 + },
  66 + "fs": {
  67 + "version": "0.0.1-security",
  68 + "resolved": "http://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz",
  69 + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ="
  70 + },
  71 + "mongodb": {
  72 + "version": "3.3.2",
  73 + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.3.2.tgz",
  74 + "integrity": "sha512-fqJt3iywelk4yKu/lfwQg163Bjpo5zDKhXiohycvon4iQHbrfflSAz9AIlRE6496Pm/dQKQK5bMigdVo2s6gBg==",
  75 + "requires": {
  76 + "bson": "^1.1.1",
  77 + "require_optional": "^1.0.1",
  78 + "safe-buffer": "^5.1.2"
  79 + }
  80 + },
  81 + "path": {
  82 + "version": "0.12.7",
  83 + "resolved": "http://registry.npmjs.org/path/-/path-0.12.7.tgz",
  84 + "integrity": "sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=",
  85 + "requires": {
  86 + "process": "^0.11.1",
  87 + "util": "^0.10.3"
  88 + },
  89 + "dependencies": {
  90 + "process": {
  91 + "version": "0.11.10",
  92 + "resolved": "http://registry.npmjs.org/process/-/process-0.11.10.tgz",
  93 + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI="
  94 + },
  95 + "util": {
  96 + "version": "0.10.3",
  97 + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz",
  98 + "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=",
  99 + "requires": {
  100 + "inherits": "2.0.1"
  101 + },
  102 + "dependencies": {
  103 + "inherits": {
  104 + "version": "2.0.1",
  105 + "resolved": "http://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
  106 + "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE="
  107 + }
  108 + }
  109 + }
  110 + }
  111 + },
  112 + "replace-ext": {
  113 + "version": "1.0.0",
  114 + "resolved": "http://registry.npmjs.org/replace-ext/-/replace-ext-1.0.0.tgz",
  115 + "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs="
  116 + },
  117 + "require_optional": {
  118 + "version": "1.0.1",
  119 + "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz",
  120 + "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==",
  121 + "requires": {
  122 + "resolve-from": "^2.0.0",
  123 + "semver": "^5.1.0"
  124 + }
  125 + },
  126 + "resolve-from": {
  127 + "version": "2.0.0",
  128 + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
  129 + "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c="
  130 + },
  131 + "safe-buffer": {
  132 + "version": "5.2.0",
  133 + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
  134 + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg=="
  135 + },
  136 + "semver": {
  137 + "version": "5.7.1",
  138 + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
  139 + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
  140 + },
  141 + "wait.for": {
  142 + "version": "0.6.6",
  143 + "resolved": "http://registry.npmjs.org/wait.for/-/wait.for-0.6.6.tgz",
  144 + "integrity": "sha1-DMkHnOBoCoBUXi7Ju3ppOhbgdQQ=",
  145 + "requires": {
  146 + "fibers": ">=1.0.1"
  147 + },
  148 + "dependencies": {
  149 + "fibers": {
  150 + "version": "2.0.0",
  151 + "resolved": "http://registry.npmjs.org/fibers/-/fibers-2.0.0.tgz",
  152 + "integrity": "sha1-8m0Krx+ZmV++HLPzQO+sCL2p3Es="
  153 + }
  154 + }
  155 + }
  156 + }
  157 +}
... ... @@ -0,0 +1,36 @@
  1 +{
  2 + "name": "extract-mongo-schema",
  3 + "version": "0.2.9",
  4 + "description": "Extract (and visualize) schema from Mongo database (including foreign keys)",
  5 + "main": "extract-mongo-schema.js",
  6 + "scripts": {
  7 + "test": "echo \"Error: no test specified\" && exit 1"
  8 + },
  9 + "repository": {
  10 + "type": "git",
  11 + "url": "https://github.com/perak/extract-mongo-schema.git"
  12 + },
  13 + "keywords": [
  14 + "mongodb",
  15 + "mongo",
  16 + "schema",
  17 + "data model",
  18 + "foreign keys",
  19 + "erm",
  20 + "er diagram"
  21 + ],
  22 + "preferGlobal": "true",
  23 + "bin": {
  24 + "extract-mongo-schema": "cli.js"
  25 + },
  26 + "author": "Petar Korponaiฤ‡ <petar.korponaic@gmail.com>",
  27 + "license": "MIT",
  28 + "dependencies": {
  29 + "command-line-args": "^3.0.5",
  30 + "fs": "0.0.1-security",
  31 + "path": "^0.12.7",
  32 + "replace-ext": "^1.0.0",
  33 + "mongodb": "^3.3.2",
  34 + "wait.for": "^0.6.6"
  35 + }
  36 +}
template-html-diagram.html
Diff comments   View file @ 8fe980d
... ... @@ -0,0 +1,202 @@
  1 +<!doctype html>
  2 +<head>
  3 + <title>Mongo Schema</title>
  4 +
  5 + <style>
  6 + svg {
  7 + background-color: white;
  8 + font-family: "Helvetic Neue", Helvetica, Arial, sans-serif;
  9 + font-size: small;
  10 + }
  11 +
  12 + .node rect,
  13 + .node circle,
  14 + .node ellipse {
  15 + stroke: #333;
  16 + opacity: 0.8;
  17 + fill: #fff;
  18 + fill-opacity: 0.6;
  19 + }
  20 + .edgePath path {
  21 + stroke: #333;
  22 + fill: #333;
  23 + fill-opacity: 1;
  24 + stroke-opacity: 1;
  25 + }
  26 + .cluster {
  27 + stroke: #999;
  28 + fill: #888;
  29 + fill-opacity: 0.3;
  30 + stroke-opacity: 0.6;
  31 + }
  32 + .entity-name rect {
  33 + fill: #08f;
  34 + fill-opacity: 0.3;
  35 + }
  36 + </style>
  37 +</head>
  38 +
  39 +<body>
  40 + <div class="background"></div>
  41 + <div class="container">
  42 + <svg></svg>
  43 + </div>
  44 + <canvas width="1024" height="1024" style="display:none"></canvas>
  45 +
  46 +
  47 + <script src="https://d3js.org/d3.v3.min.js"></script>
  48 + <script src="https://cdnjs.cloudflare.com/ajax/libs/dagre-d3/0.4.17/dagre-d3.min.js"></script>
  49 + <script src="https://code.jquery.com/jquery-3.2.1.min.js" integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4=" crossorigin="anonymous"></script>
  50 +
  51 + <script>
  52 + var drawERM = function(data) {
  53 + var width = window.innerWidth,
  54 + initialHeight = window.innerHeight,
  55 + svg = d3.select("body div.container svg"),
  56 + inner = svg.append("g");
  57 +
  58 + svg.attr('width', width).attr('height', initialHeight);
  59 +
  60 + var resizeGraph = function(doCenter) {
  61 + var newWidth = window.innerWidth;
  62 + var newHeight = window.innerHeight;
  63 +
  64 + if(newWidth < g.graph().width + 40) {
  65 + newWidth = g.graph().width + 40;
  66 + }
  67 + if(newHeight < g.graph().height + 40) {
  68 + newHeight = g.graph().height + 40;
  69 + }
  70 + svg.attr('width', newWidth).attr('height', newHeight);
  71 +
  72 + // Center the graph
  73 + if(doCenter) {
  74 + zoom
  75 + .translate([(svg.attr("width") - g.graph().width * initialScale) / 2, 20])
  76 + .scale(initialScale)
  77 + .event(svg);
  78 + }
  79 + }
  80 +
  81 + // Set up zoom support
  82 + var zoom = d3.behavior.zoom().on("zoom", function() {
  83 + inner.attr("transform", "translate(" + d3.event.translate + ")" +
  84 + "scale(" + d3.event.scale + ")");
  85 + });
  86 + svg.call(zoom);
  87 +
  88 + // create graph
  89 + var g = new dagreD3.graphlib.Graph({
  90 + multigraph: false,
  91 + compound: true
  92 + }).setGraph({
  93 + rankdir: "LR",
  94 + edgesep: 25,
  95 + nodesep: 0
  96 + });
  97 +
  98 + var links = [];
  99 +
  100 + var addField = function(collectionName, fieldName, fieldInfo, parentFieldName) {
  101 + var nodeName = parentFieldName ? collectionName + "_" + parentFieldName + "_" + fieldName : collectionName + "_" + fieldName;
  102 + g.setNode(nodeName, {
  103 + label: (parentFieldName ? parentFieldName + "." + fieldName : fieldName) + ": " + fieldInfo.type,
  104 + width: 300
  105 + });
  106 + g.setParent(nodeName, "___" + collectionName + "_container");
  107 +
  108 + if(fieldInfo.foreignKey && fieldInfo.references) {
  109 + links.push({
  110 + from: nodeName,
  111 + to: fieldInfo.references + "__id"
  112 + });
  113 + }
  114 +
  115 + if(fieldInfo.type == "Object" && fieldInfo.structure) {
  116 + for(var subFieldName in fieldInfo.structure) {
  117 + addField(collectionName, subFieldName, fieldInfo.structure[subFieldName], parentFieldName ? parentFieldName + "." + fieldName : fieldName);
  118 + }
  119 + }
  120 + };
  121 +
  122 + var addCollection = function(collectionName, collectionInfo) {
  123 + g.setNode("___" + collectionName + "_container", {
  124 + label: ""
  125 + });
  126 + g.setNode("___" + collectionName + "_title", {
  127 + label: collectionName,
  128 + class: 'entity-name',