import { Box } from '../common/box'; import { Cluster } from '../common/cluster'; import { HSV, hsv2rgb, RGB } from '../common/hsv2rgb'; import { ImageDebug } from '../common/image-debug'; import { ImageWrapper } from '../common/image-wrapper'; import { Moment } from '../common/moment'; import { Point } from '../common/point'; import { calculatePatchSize } from '../input/input-stream-utils'; import { BarcodeLocatorConfig } from './barcode-locator-config'; import { halfSample, invert, otsuThreshold, transformWithMatrix } from './barcode-locator-utils'; import { Rasterizer } from './rasterizer'; import skeletonizer from './skeletonizer'; import { SearchDirections } from './tracer'; interface Patch { box: Box; index: number; moments: Array; pos: Point; rad: number; x: number; y: number; } type Sceletonizer = any; const MomentSimilarityThreshold = 0.9; export class BarcodeLocator { private _config: BarcodeLocatorConfig; private _inputImageWrapper: ImageWrapper; private _currentImageWrapper: ImageWrapper; private _skelImageWrapper: ImageWrapper; private _subImageWrapper: ImageWrapper; private _labelImageWrapper: ImageWrapper>; private _binaryImageWrapper: ImageWrapper; private _patchGrid: ImageWrapper; private _patchLabelGrid: ImageWrapper; private _imageToPatchGrid: Array; private _patchSize: Point; private _binaryContext: CanvasRenderingContext2D; private _numPatches: Point; private _skeletonizer: Sceletonizer; constructor(inputImageWrapper: ImageWrapper, config: BarcodeLocatorConfig) { this._config = config; this._inputImageWrapper = inputImageWrapper; this._numPatches = { x: 0, y: 0 }; this._initBuffers(); this._initCanvas(); } locate() { if (this._config.halfSample) { halfSample(this._inputImageWrapper, this._currentImageWrapper); } this._binarizeImage(); const patchesFound = this._findPatches(); // return unless 5% or more patches are found if (patchesFound.length < this._numPatches.x * this._numPatches.y * 0.05) { return null; } // rasterize area by comparing angular similarity; const maxLabel = this._rasterizeAngularSimilarity(patchesFound); if (maxLabel < 1) { return null; } // search for area with the most patches (biggest connected area) const topLabels = this._findBiggestConnectedAreas(maxLabel); if (topLabels.length === 0) { return null; } const boxes = this._findBoxes(topLabels, maxLabel); return boxes; } private _initBuffers(): void { if (this._config.halfSample) { this._currentImageWrapper = new ImageWrapper({ x: this._inputImageWrapper.size.x / 2 | 0, y: this._inputImageWrapper.size.y / 2 | 0 }); } else { this._currentImageWrapper = this._inputImageWrapper; } this._patchSize = calculatePatchSize(this._config.patchSize, this._currentImageWrapper.size); this._numPatches.x = this._currentImageWrapper.size.x / this._patchSize.x | 0; this._numPatches.y = this._currentImageWrapper.size.y / this._patchSize.y | 0; this._binaryImageWrapper = new ImageWrapper(this._currentImageWrapper.size, undefined, Uint8Array, false); this._labelImageWrapper = new ImageWrapper(this._patchSize, undefined, Array, true); const skeletonImageData = new ArrayBuffer(64 * 1024); this._subImageWrapper = new ImageWrapper(this._patchSize, new Uint8Array(skeletonImageData, 0, this._patchSize.x * this._patchSize.y)); this._skelImageWrapper = new ImageWrapper(this._patchSize, new Uint8Array(skeletonImageData, this._patchSize.x * this._patchSize.y * 3, this._patchSize.x * this._patchSize.y), undefined, true); this._skeletonizer = skeletonizer( (typeof window !== 'undefined') ? window : (typeof self !== 'undefined') ? self : global, { size: this._patchSize.x }, skeletonImageData ); const size = { x: (this._currentImageWrapper.size.x / this._subImageWrapper.size.x) | 0, y: (this._currentImageWrapper.size.y / this._subImageWrapper.size.y) | 0 }; this._patchLabelGrid = new ImageWrapper(size, undefined, Int32Array, true); this._patchGrid = new ImageWrapper(size, undefined, undefined, true); this._imageToPatchGrid = new Array(this._patchLabelGrid.data.length); } private _initCanvas() { if (this._config.useWorker || typeof document === 'undefined') { return; } const canvas = document.createElement('canvas'); canvas.className = 'binaryBuffer'; canvas.width = this._binaryImageWrapper.size.x; canvas.height = this._binaryImageWrapper.size.y; if (process.env.NODE_ENV !== 'production' && this._config.debug && this._config.debug.showCanvas) { document.querySelector('#debug').appendChild(canvas); } this._binaryContext = canvas.getContext('2d'); } /** * Creates a bounding box which encloses all the given patches * @returns The minimal bounding box */ private _boxFromPatches(patches: Array): Box { const debug = process.env.NODE_ENV !== 'production' && this._config.debug; let averageRad = patches.reduce((sum, { pos, rad }) => { if (debug && debug.showPatches) { // draw all patches which are to be taken into consideration this._drawRect(pos, this._subImageWrapper.size, 'red', 1); } return sum + rad; }, 0) / patches.length; averageRad = (averageRad * 180 / Math.PI + 90) % 180 - 90; if (averageRad < 0) { averageRad += 180; } averageRad = (180 - averageRad) * Math.PI / 180; const cos = Math.cos(averageRad); const sin = Math.sin(averageRad); const matrix = new Float32Array([cos, sin, -sin, cos]); const inverseMatrix = invert(matrix); // iterate over patches and rotate by angle patches.forEach(({ box }) => { for (let j = 0; j < 4; j++) { box[j] = transformWithMatrix(box[j], matrix); } if (debug && debug.boxFromPatches.showTransformed) { this._drawPath(box, '#99ff00', 2); } }); let minX = this._binaryImageWrapper.size.x; let minY = this._binaryImageWrapper.size.y; let maxX = -minX; let maxY = -minY; // find bounding box patches.forEach(({ box }) => { box.forEach(({ x, y }) => { if (x < minX) { minX = x; } if (x > maxX) { maxX = x; } if (y < minY) { minY = y; } if (y > maxY) { maxY = y; } }); }); let box: Box = [{ x: minX, y: minY }, { x: maxX, y: minY }, { x: maxX, y: maxY }, { x: minX, y: maxY }]; if (debug && debug.boxFromPatches.showTransformedBox) { this._drawPath(box, '#ff0000', 2); } // reverse rotation box = box.map(vertex => transformWithMatrix(vertex, inverseMatrix)) as Box; if (debug && debug.boxFromPatches.showBB) { this._drawPath(box, '#ff0000', 2); } if (this._config.halfSample) { // scale box = box.map(({ x, y }) => ({ x: x * 2, y: y *= 2 })) as Box; } return box; } /** * Creates a binary image of the current image */ private _binarizeImage(): void { otsuThreshold(this._currentImageWrapper, this._binaryImageWrapper); this._binaryImageWrapper.zeroBorder(); if (process.env.NODE_ENV !== 'production' && this._config.debug && this._config.debug.showCanvas) { this._binaryImageWrapper.show(this._binaryContext, 255); } } /** * Iterate over the entire image, extract patches */ private _findPatches(): Array { const debug = process.env.NODE_ENV !== 'production' && this._config.debug; let patchesFound = new Array(); for (let i = 0; i < this._numPatches.x; i++) { for (let j = 0; j < this._numPatches.y; j++) { const x = this._subImageWrapper.size.x * i; const y = this._subImageWrapper.size.y * j; // seperate parts this._skeletonize(x, y); // Rasterize, find individual bars this._skelImageWrapper.zeroBorder(); this._labelImageWrapper.data.fill(0); const rasterizer = new Rasterizer(this._skelImageWrapper, this._labelImageWrapper); const rasterResult = rasterizer.rasterize(0); if (debug && debug.showLabels) { this._labelImageWrapper.overlay(this._binaryContext, 360 / rasterResult.count | 0, x, y); } // calculate moments from the skeletonized patch const moments = this._labelImageWrapper.moments(rasterResult.count); // extract eligible patches const patch = this._describePatch(moments, j * this._numPatches.x + i, x, y); if (patch) { patchesFound.push(patch); if (debug && debug.showFoundPatches) { this._drawRect(patch.pos, this._subImageWrapper.size, '#99ff00', 2); } } } } return patchesFound; } /** * Finds those connected areas which contain at least 6 patches * and returns them ordered DESC by the number of contained patches * @param maxLabel */ private _findBiggestConnectedAreas(maxLabel: number): Array { let labelHist = new Array(maxLabel).fill(0); this._patchLabelGrid.data.forEach((data: number) => { if (data > 0) { labelHist[data - 1]++; } }); // extract top areas with at least 6 patches present const topLabels = labelHist.map((value, index) => ({ value, index })) .filter(({ value }) => value >= 5).sort((a, b) => b.value - a.value).map(({ index }) => index + 1); return topLabels; } /** * */ private _findBoxes(topLabels: Array, maxLabel: number): Array { const boxes = new Array(); const showRemainingPatchLabels = process.env.NODE_ENV !== 'production' && this._config.debug && this._config.debug.showRemainingPatchLabels; topLabels.forEach(label => { const patches = new Array(); this._patchLabelGrid.data.forEach((data: number, index: number) => { if (data === label) { patches.push(this._imageToPatchGrid[index]); } }); const box = this._boxFromPatches(patches); if (box) { boxes.push(box); if (showRemainingPatchLabels) { // draw patch-labels if requested const hsv: HSV = [(label / (maxLabel + 1)) * 360, 1, 1]; const rgb: RGB = [0, 0, 0]; hsv2rgb(hsv, rgb); const color = `rgb(${rgb.join(',')})`; patches.forEach(({ pos }) => this._drawRect(pos, this._subImageWrapper.size, color, 2)); } } }); return boxes; } /** * Find similar moments (via cluster) * @param moments */ private _similarMoments(moments: Array): Array { const clusters = Cluster.clusterize(moments, MomentSimilarityThreshold); const topCluster = clusters.reduce((top, item) => { const count = item.moments.length; return count > top.count ? { item, count } : top; }, { item: { moments: [] }, count: 0 }); const result = topCluster.item.moments; return result; } private _skeletonize(x: number, y: number): void { this._binaryImageWrapper.subImageAsCopy(this._subImageWrapper, x, y); this._skeletonizer.skeletonize(); // Show skeleton if requested if (process.env.NODE_ENV !== 'production' && this._config.debug && this._config.debug.showSkeleton) { this._skelImageWrapper.overlay(this._binaryContext, 360, x, y); } } /** * Extracts and describes those patches which seem to contain a barcode pattern * @param moments * @param index * @param x * @param y * @returns list of patches */ private _describePatch(moments: Array, index: number, x: number, y: number): Patch { if (moments.length > 1) { const minComponentWeight = Math.ceil(this._patchSize.x / 3); // only collect moments which area covers at least minComponentWeight pixels const eligibleMoments = moments.filter(moment => moment.m00 > minComponentWeight); // if at least 2 moments are found which have at least minComponentWeights covered if (eligibleMoments.length > 1) { const matchingMoments = this._similarMoments(eligibleMoments); const length = matchingMoments.length | 0; // Only two of the moments are allowed not to fit into the equation if (length > 1 && (length << 2) >= eligibleMoments.length * 3 && (length << 2) > moments.length) { // determine the similarity of the moments const rad = matchingMoments.reduce((sum: number, moment: Moment) => sum + moment.rad, 0) / length; return { index, pos: { x, y }, box: [ { x, y }, { x: x + this._subImageWrapper.size.x, y }, { x: x + this._subImageWrapper.size.x, y: y + this._subImageWrapper.size.y }, { x, y: y + this._subImageWrapper.size.y } ], moments: matchingMoments, rad, x: Math.cos(rad), y: Math.sin(rad) }; } } } return null; } private _notYetProcessed(): number { for (let i = 0; i < this._patchLabelGrid.data.length; i++) { if (this._patchLabelGrid.data[i] === 0 && this._patchGrid.data[i] === 1) { return i; } } return this._patchLabelGrid.data.length; } private _trace(currentIndex: number, label: number): void { const threshold = 0.95; const current: Point = { x: currentIndex % this._patchLabelGrid.size.x, y: (currentIndex / this._patchLabelGrid.size.x) | 0 }; if (currentIndex < this._patchLabelGrid.data.length) { const currentPatch = this._imageToPatchGrid[currentIndex]; // assign label this._patchLabelGrid.data[currentIndex] = label; SearchDirections.forEach(direction => { const y = current.y + direction[0]; const x = current.x + direction[1]; const index = y * this._patchLabelGrid.size.x + x; // continue if patch empty if (this._patchGrid.data[index] === 0) { this._patchLabelGrid.data[index] = Number.MAX_VALUE; } else if (this._patchLabelGrid.data[index] === 0) { const patch = this._imageToPatchGrid[index]; const similarity = Math.abs(patch.x * currentPatch.x + patch.y * currentPatch.y); if (similarity > threshold) { this._trace(index, label); } } }); } } /** * Finds patches which are connected and share the same orientation * @param patchesFound */ private _rasterizeAngularSimilarity(patchesFound: Array): number { let label = 0; const hsv: HSV = [0, 1, 1]; const rgb: RGB = [0, 0, 0]; // prepare for finding the right patches this._patchGrid.data.fill(0); this._patchLabelGrid.data.fill(0); this._imageToPatchGrid.fill(null); patchesFound.forEach(patch => { this._imageToPatchGrid[patch.index] = patch; this._patchGrid.data[patch.index] = 1; }); // rasterize the patches found to determine area this._patchGrid.zeroBorder(); let currentIndex = 0; while ((currentIndex = this._notYetProcessed()) < this._patchLabelGrid.data.length) { label++; this._trace(currentIndex, label); } // draw patch-labels if requested if (process.env.NODE_ENV !== 'production' && this._config.debug && this._config.debug.showPatchLabels) { for (let j = 0; j < this._patchLabelGrid.data.length; j++) { if (this._patchLabelGrid.data[j] > 0 && this._patchLabelGrid.data[j] <= label) { const patch = this._imageToPatchGrid[j]; hsv[0] = (this._patchLabelGrid.data[j] / (label + 1)) * 360; hsv2rgb(hsv, rgb); this._drawRect(patch.pos, this._subImageWrapper.size, `rgb(${rgb.join(',')})`, 2); } } } return label; } private _drawRect({ x, y }: Point, size: Point, color: string, lineWidth: number): void { this._binaryContext.strokeStyle = color; this._binaryContext.fillStyle = color; this._binaryContext.lineWidth = lineWidth || 1; this._binaryContext.strokeRect(x, y, size.x, size.y); } private _drawPath(path: Array, color: string, lineWidth: number): void { ImageDebug.drawPath(path, this._binaryContext, color, lineWidth); } }