diff --git a/src/geometry/__tests__/getPerspectiveWarp.test.ts b/src/geometry/__tests__/getPerspectiveWarp.test.ts new file mode 100644 index 000000000..f96ede244 --- /dev/null +++ b/src/geometry/__tests__/getPerspectiveWarp.test.ts @@ -0,0 +1,218 @@ +import { Image } from '../../Image.js'; +import getPerspectiveWarp, { order4Points } from '../getPerspectiveWarp.js'; + +describe('4 points sorting', () => { + test('basic sorting test', () => { + const points = [ + { column: 0, row: 100 }, + { column: 0, row: 0 }, + { column: 100, row: 1 }, + { column: 100, row: 100 }, + ]; + + const result = order4Points(points); + expect(result).toEqual([ + { column: 0, row: 0 }, + { column: 100, row: 1 }, + { column: 100, row: 100 }, + { column: 0, row: 100 }, + ]); + }); + test('inclined square', () => { + const points = [ + { column: 45, row: 0 }, + { column: 0, row: 45 }, + { column: 45, row: 90 }, + { column: 90, row: 45 }, + ]; + + const result = order4Points(points); + expect(result).toEqual([ + { column: 0, row: 45 }, + { column: 90, row: 45 }, + { column: 45, row: 0 }, + { column: 45, row: 90 }, + ]); + }); + test('basic sorting test', () => { + const points = [ + { column: 155, row: 195 }, + { column: 154, row: 611 }, + { column: 858.5, row: 700 }, + { column: 911.5, row: 786 }, + ]; + + const result = order4Points(points); + expect(result).toEqual([ + { column: 155, row: 195 }, + + { column: 858.5, row: 700 }, + { column: 911.5, row: 786 }, + { column: 154, row: 611 }, + ]); + }); +}); + +describe('warping tests', () => { + it('resize without rotation', () => { + const image = new Image(3, 3, { + data: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9]), + colorModel: 'GREY', + }); + const points = [ + { column: 0, row: 0 }, + { column: 2, row: 0 }, + { column: 1, row: 2 }, + { column: 0, row: 2 }, + ]; + const matrix = getPerspectiveWarp(points); + const result = image.transform(matrix.matrix, { inverse: true }); + expect(result.width).not.toBeLessThan(2); + expect(result.height).not.toBeLessThan(2); + expect(result.width).not.toBeGreaterThan(3); + expect(result.height).not.toBeGreaterThan(3); + }); + it('resize without rotation 2', () => { + const image = new Image(4, 4, { + data: new Uint8Array([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ]), + colorModel: 'GREY', + }); + + const points = [ + { column: 0, row: 0 }, + { column: 3, row: 0 }, + { column: 2, row: 1 }, + { column: 0, row: 1 }, + ]; + const matrix = getPerspectiveWarp(points); + const result = image.transform(matrix.matrix, { inverse: true }); + expect(result.width).not.toBeLessThan(3); + expect(result.height).not.toBeLessThan(1); + expect(result.width).not.toBeGreaterThan(4); + expect(result.height).not.toBeGreaterThan(4); + }); +}); + +describe('openCV comparison', () => { + test('nearest interpolation plants', () => { + const image = testUtils.load('various/plants.png'); + const openCvResult = testUtils.load( + 'opencv/test_perspective_warp_plants_nearest.png', + ); + + const points = [ + { column: 858.5, row: 9 }, + { column: 911.5, row: 786 }, + { column: 154.5, row: 611 }, + { column: 166.5, row: 195 }, + ]; + const matrix = getPerspectiveWarp(points, { + width: 1080, + height: 810, + }); + const result = image.transform(matrix.matrix, { + inverse: true, + interpolationType: 'nearest', + }); + const croppedPieceOpenCv = openCvResult.crop({ + origin: { column: 45, row: 0 }, + width: 100, + height: 100, + }); + + const croppedPiece = result.crop({ + origin: { column: 45, row: 0 }, + width: 100, + height: 100, + }); + + expect(result.width).toEqual(openCvResult.width); + expect(result.height).toEqual(openCvResult.height); + expect(croppedPiece).toEqual(croppedPieceOpenCv); + }); + + test('nearest interpolation card', () => { + const image = testUtils.load('various/card.png'); + const openCvResult = testUtils.load( + 'opencv/test_perspective_warp_card_nearest.png', + ); + const points = [ + { column: 55, row: 140 }, + { column: 680, row: 38 }, + { column: 840, row: 340 }, + { column: 145, row: 460 }, + ]; + const matrix = getPerspectiveWarp(points, { + width: 700, + height: 400, + }); + const result = image.transform(matrix.matrix, { + inverse: true, + interpolationType: 'nearest', + width: 700, + height: 400, + }); + const croppedPieceOpenCv = openCvResult.crop({ + origin: { column: 45, row: 0 }, + width: 5, + height: 5, + }); + + const croppedPiece = result.crop({ + origin: { column: 45, row: 0 }, + width: 5, + height: 5, + }); + + expect(result.width).toEqual(openCvResult.width); + expect(result.height).toEqual(openCvResult.height); + expect(croppedPiece).toEqual(croppedPieceOpenCv); + }); + test('nearest interpolation poker card', () => { + const image = testUtils.load('various/poker_cards.png'); + const openCvResult = testUtils.load( + 'opencv/test_perspective_warp_poker_cards_nearest.png', + ); + + const points = [ + { column: 1100, row: 660 }, + { column: 680, row: 660 }, + { column: 660, row: 290 }, + { column: 970, row: 290 }, + ]; + const matrix = getPerspectiveWarp(points); + const result = image.transform(matrix.matrix, { + inverse: true, + interpolationType: 'nearest', + height: matrix.height, + width: matrix.width, + }); + + const cropped = result.crop({ + origin: { column: 10, row: 10 }, + width: 100, + height: 100, + }); + const croppedCV = openCvResult.crop({ + origin: { column: 10, row: 10 }, + width: 100, + height: 100, + }); + + expect(result.width).toEqual(openCvResult.width); + expect(result.height).toEqual(openCvResult.height); + expect(cropped).toEqual(croppedCV); + }); +}); + +describe('error testing', () => { + test("should throw if there aren't 4 points", () => { + expect(() => { + getPerspectiveWarp([{ column: 1, row: 1 }]); + }).toThrow( + 'The array pts must have four elements, which are the four corners. Currently, pts have 1 elements', + ); + }); +}); diff --git a/src/geometry/__tests__/transform.test.ts b/src/geometry/__tests__/transform.test.ts index 9aab15089..9a49acdba 100644 --- a/src/geometry/__tests__/transform.test.ts +++ b/src/geometry/__tests__/transform.test.ts @@ -121,5 +121,5 @@ test('should throw if matrix has wrong size', () => { ]; expect(() => { img.transform(translation); - }).toThrow('transformation matrix must be 2x3. Received 2x4'); + }).toThrow('transformation matrix must be 2x3 or 3x3. Received 2x4'); }); diff --git a/src/geometry/getPerspectiveWarp.ts b/src/geometry/getPerspectiveWarp.ts new file mode 100644 index 000000000..0f7f542fb --- /dev/null +++ b/src/geometry/getPerspectiveWarp.ts @@ -0,0 +1,161 @@ +import { Matrix, SingularValueDecomposition } from 'ml-matrix'; + +import type { Point } from '../utils/geometry/points.js'; + +interface GetPerspectiveWarpOptions { + /** + * The horizontal dimension (in pixels) of the final rectified rectangular image. + */ + width?: number; + /** + * The vertical dimension (in pixels) of the final rectified rectangular image. + */ + height?: number; +} +/** + * Returns result matrix along with vertical and horizontal dimensions for the rectangular image. + */ +type GetPerspectiveWarpData = Required & { + matrix: number[][]; +}; + +// REFERENCES : +// https://stackoverflow.com/questions/38285229/calculating-aspect-ratio-of-perspective-transform-destination-image/38402378#38402378 +// http://www.corrmap.com/features/homography_transformation.php +// https://ags.cs.uni-kl.de/fileadmin/inf_ags/3dcv-ws11-12/3DCV_WS11-12_lec04.pdf +// http://graphics.cs.cmu.edu/courses/15-463/2011_fall/Lectures/morphing.pdf +/** + * Returns perspective warp matrix from 4 points. + * @param pts - 4 reference corners of the new image. + * @param options - PerspectiveWarpOptions + * @returns - Matrix from 4 points. + */ +export default function getPerspectiveWarp( + pts: Point[], + options: GetPerspectiveWarpOptions = {}, +): GetPerspectiveWarpData { + if (pts.length !== 4) { + throw new Error( + `The array pts must have four elements, which are the four corners. Currently, pts have ${pts.length} elements`, + ); + } + const { width, height } = options; + const [tl, tr, br, bl] = order4Points(pts); + + let widthRect; + let heightRect; + if (height && width) { + widthRect = width; + heightRect = height; + } else { + widthRect = Math.ceil( + Math.max(distance2Points(tl, tr), distance2Points(bl, br)), + ); + heightRect = Math.ceil( + Math.max(distance2Points(tl, bl), distance2Points(tr, br)), + ); + } + + const [x1, y1] = [0, 0]; + const [x2, y2] = [widthRect - 1, 0]; + const [x3, y3] = [widthRect - 1, heightRect - 1]; + const [x4, y4] = [0, heightRect - 1]; + + const S = new Matrix([ + [x1, y1, 1, 0, 0, 0, -x1 * tl.column, -y1 * tl.column], + [x2, y2, 1, 0, 0, 0, -x2 * tr.column, -y2 * tr.column], + [x3, y3, 1, 0, 0, 0, -x3 * br.column, -y3 * br.column], + [x4, y4, 1, 0, 0, 0, -x4 * bl.column, -y4 * bl.column], + [0, 0, 0, x1, y1, 1, -x1 * tl.row, -y1 * tl.row], + [0, 0, 0, x2, y2, 1, -x2 * tr.row, -y2 * tr.row], + [0, 0, 0, x3, y3, 1, -x3 * br.row, -y3 * br.row], + [0, 0, 0, x4, y4, 1, -x4 * bl.row, -y4 * bl.row], + ]); + const D = Matrix.columnVector([ + tl.column, + tr.column, + br.column, + bl.column, + tl.row, + tr.row, + br.row, + bl.row, + ]); + + const svd = new SingularValueDecomposition(S); + const T = svd.solve(D).to1DArray(); // solve S*T = D + T.push(1); + + const M = []; + for (let i = 0; i < 3; i++) { + const row = []; + for (let j = 0; j < 3; j++) { + row.push(T[i * 3 + j]); + } + M.push(row); + } + return { matrix: M, width: widthRect, height: heightRect }; +} + +/** + * Sorts 4 points in order =>[top-left,top-right,bottom-right,bottom-left]. Input points must be in clockwise or counter-clockwise order. + * @param pts - Array of 4 points. + * @returns Sorted array of 4 points. + */ +export function order4Points(pts: Point[]) { + let tl: Point; + let tr: Point; + let br: Point; + let bl: Point; + + let minX = pts[0].column; + let indexMinX = 0; + + for (let i = 1; i < pts.length; i++) { + if (pts[i].column < minX) { + minX = pts[i].column; + indexMinX = i; + } + } + + let minX2 = pts[(indexMinX + 1) % pts.length].column; + let indexMinX2 = (indexMinX + 1) % pts.length; + + for (let i = 0; i < pts.length; i++) { + if (pts[i].column < minX2 && i !== indexMinX) { + minX2 = pts[i].column; + indexMinX2 = i; + } + } + if (pts[indexMinX2].row < pts[indexMinX].row) { + tl = pts[indexMinX2]; + bl = pts[indexMinX]; + if (indexMinX !== (indexMinX2 + 1) % 4) { + tr = pts[(indexMinX2 + 1) % 4]; + br = pts[(indexMinX2 + 2) % 4]; + } else { + tr = pts[(indexMinX2 + 2) % 4]; + br = pts[(indexMinX2 + 3) % 4]; + } + } else { + bl = pts[indexMinX2]; + tl = pts[indexMinX]; + if (indexMinX2 !== (indexMinX + 1) % 4) { + tr = pts[(indexMinX + 1) % 4]; + br = pts[(indexMinX + 2) % 4]; + } else { + tr = pts[(indexMinX + 2) % 4]; + br = pts[(indexMinX + 3) % 4]; + } + } + return [tl, tr, br, bl]; +} +/** + * Calculates distance between points. + * @param p1 - Point1 + * @param p2 - Point2 + * @returns distance between points. + */ +function distance2Points(p1: Point, p2: Point) { + return Math.hypot(p1.column - p2.column, p1.row - p2.row); +} diff --git a/src/geometry/transform.ts b/src/geometry/transform.ts index ccd08d833..ebf8ad932 100644 --- a/src/geometry/transform.ts +++ b/src/geometry/transform.ts @@ -62,7 +62,6 @@ export function transform( fullImage, } = options; let { width = image.width, height = image.height } = options; - if (fullImage) { transformMatrix = transformMatrix.map((row) => row.slice()); transformMatrix[0][2] = 0; @@ -81,8 +80,18 @@ export function transform( const transformedCorners = corners.map((corner) => { return [ - transformPoint(transformMatrix[0], corner.column, corner.row), - transformPoint(transformMatrix[1], corner.column, corner.row), + transformPoint( + transformMatrix[0], + transformMatrix[2], + corner.column, + corner.row, + ), + transformPoint( + transformMatrix[1], + transformMatrix[2], + corner.column, + corner.row, + ), ]; }); @@ -97,8 +106,18 @@ export function transform( width = maxX - minX; height = maxY - minY; - const centerX = transformPoint(transformMatrix[0], center[0], center[1]); - const centerY = transformPoint(transformMatrix[1], center[0], center[1]); + const centerX = transformPoint( + transformMatrix[0], + transformMatrix[2], + center[0], + center[1], + ); + const centerY = transformPoint( + transformMatrix[1], + transformMatrix[2], + center[0], + center[1], + ); const a = (width - 1) / 2 - centerX; const b = (height - 1) / 2 - centerY; transformMatrix[0][2] = a; @@ -107,18 +126,16 @@ export function transform( height = Math.round(height); } - if ( - transformMatrix.length !== 2 || - transformMatrix[0].length !== 3 || - transformMatrix[1].length !== 3 - ) { + if (!isValidMatrix(transformMatrix)) { throw new TypeError( - `transformation matrix must be 2x3. Received ${transformMatrix.length}x${transformMatrix[1].length}`, + `transformation matrix must be 2x3 or 3x3. Received ${transformMatrix.length}x${transformMatrix[1].length}`, ); } + if (transformMatrix.length === 2) { + transformMatrix.push([0, 0, 1]); + } if (!options.inverse) { - transformMatrix = [transformMatrix[0], transformMatrix[1], [0, 0, 1]]; transformMatrix = inverse(new Matrix(transformMatrix)).to2DArray(); } const newImage = Image.createFrom(image, { @@ -132,8 +149,18 @@ export function transform( const interpolate = getInterpolationFunction(interpolationType); for (let row = 0; row < newImage.height; row++) { for (let column = 0; column < newImage.width; column++) { - const nx = transformPoint(transformMatrix[0], column, row); - const ny = transformPoint(transformMatrix[1], column, row); + const nx = transformPoint( + transformMatrix[0], + transformMatrix[2], + column, + row, + ); + const ny = transformPoint( + transformMatrix[1], + transformMatrix[2], + column, + row, + ); for (let channel = 0; channel < newImage.channels; channel++) { const newValue = interpolate( image, @@ -154,14 +181,31 @@ export function transform( /** * Apply a transformation to a point. * @param transform - Transformation matrix. + * @param perspective - Perspective matrix. * @param column - Column of the point. * @param row - Row of the point. * @returns New value. */ function transformPoint( transform: number[], + perspective: number[], column: number, row: number, ): number { - return transform[0] * column + transform[1] * row + transform[2]; + return ( + (transform[0] * column + transform[1] * row + transform[2]) / + (perspective[0] * column + perspective[1] * row + perspective[2]) + ); +} + +function isValidMatrix(transformationMatrix: number[][]) { + return ( + (transformationMatrix.length === 3 && + transformationMatrix[0].length === 3 && + transformationMatrix[1].length === 3 && + transformationMatrix[2].length === 3) || + (transformationMatrix.length === 2 && + transformationMatrix[0].length === 3 && + transformationMatrix[1].length === 3) + ); } diff --git a/src/geometry/transformRotate.ts b/src/geometry/transformRotate.ts index 86875c080..a1a672de1 100644 --- a/src/geometry/transformRotate.ts +++ b/src/geometry/transformRotate.ts @@ -47,7 +47,7 @@ export function transformRotate( * @param angle - Angle in degrees. * @param center - Center point of the image. * @param scale - Scaling factor. - * @returns 2 x 3 rotation matrix. + * @returns 3 x 3 rotation matrix. */ function getRotationMatrix( angle: number, @@ -60,5 +60,6 @@ function getRotationMatrix( return [ [cos, sin, (1 - cos) * center.column - sin * center.row], [-sin, cos, sin * center.column + (1 - cos) * center.row], + [0, 0, 1], ]; } diff --git a/test/TestImagePath.ts b/test/TestImagePath.ts index 0cab6ec39..9268d9e98 100644 --- a/test/TestImagePath.ts +++ b/test/TestImagePath.ts @@ -74,6 +74,10 @@ export type TestImagePath = | 'opencv/testConvolution.png' | 'opencv/testGaussianBlur.png' | 'opencv/testInterpolate.png' + | 'opencv/test_perspective_warp_card_nearest.png' + | 'opencv/test_perspective_warp_plants_linear.png' + | 'opencv/test_perspective_warp_plants_nearest.png' + | 'opencv/test_perspective_warp_poker_cards_nearest.png' | 'opencv/testReflect.png' | 'opencv/test_resize_bicubic_larger.png' | 'opencv/test_resize_bicubic_same.png' @@ -94,7 +98,10 @@ export type TestImagePath = | 'ssim/ssim-original.png' | 'ssim/ssim-saltPepper.png' | 'various/alphabet.jpg' + | 'various/card.png' | 'various/grayscale_by_zimmyrose.png' + | 'various/plants.png' + | 'various/poker_cards.png' | 'various/screws.png' | 'various/sudoku.jpg' | 'various/without-metadata.jpg'; diff --git a/test/img/opencv/test_perspective_warp_card_nearest.png b/test/img/opencv/test_perspective_warp_card_nearest.png new file mode 100644 index 000000000..2a6fe9d1f Binary files /dev/null and b/test/img/opencv/test_perspective_warp_card_nearest.png differ diff --git a/test/img/opencv/test_perspective_warp_plants_nearest.png b/test/img/opencv/test_perspective_warp_plants_nearest.png new file mode 100644 index 000000000..ecc37b810 Binary files /dev/null and b/test/img/opencv/test_perspective_warp_plants_nearest.png differ diff --git a/test/img/opencv/test_perspective_warp_poker_cards_nearest.png b/test/img/opencv/test_perspective_warp_poker_cards_nearest.png new file mode 100644 index 000000000..5d8a61ae2 Binary files /dev/null and b/test/img/opencv/test_perspective_warp_poker_cards_nearest.png differ diff --git a/test/img/various/card.png b/test/img/various/card.png new file mode 100644 index 000000000..f3e1b95a8 Binary files /dev/null and b/test/img/various/card.png differ diff --git a/test/img/various/plants.png b/test/img/various/plants.png new file mode 100644 index 000000000..5e6091113 Binary files /dev/null and b/test/img/various/plants.png differ diff --git a/test/img/various/poker_cards.png b/test/img/various/poker_cards.png new file mode 100644 index 000000000..388b0d0de Binary files /dev/null and b/test/img/various/poker_cards.png differ