From 8b07cbb614d2b4425070aeeba98bef8debc07cf0 Mon Sep 17 00:00:00 2001 From: Paul Novotny Date: Sun, 14 Jul 2024 09:38:08 -0400 Subject: [PATCH 1/4] Add 3D rotations This adds 3D support to the Rotate transformation, and introduces RotateX, RotateY, and RotateZ for 3D rotations around the x, y, and z axes. --- src/DataAugmentation.jl | 3 ++ src/projective/affine.jl | 80 +++++++++++++++++++++++++++++++++------ test/projective/affine.jl | 55 ++++++++++++++++++++++++++- 3 files changed, 124 insertions(+), 14 deletions(-) diff --git a/src/DataAugmentation.jl b/src/DataAugmentation.jl index 419dd418..4b1c1e50 100644 --- a/src/DataAugmentation.jl +++ b/src/DataAugmentation.jl @@ -70,6 +70,9 @@ export Item, RandomCrop, ScaleFixed, Rotate, + RotateX, + RotateY, + RotateZ, RandomResizeCrop, CenterResizeCrop, Buffered, diff --git a/src/projective/affine.jl b/src/projective/affine.jl index 3205d414..0c77695e 100644 --- a/src/projective/affine.jl +++ b/src/projective/affine.jl @@ -120,10 +120,14 @@ end """ Rotate(γ) - Rotate(γs) + Rotate(distribution) + Rotate(α, β, γ) + Rotate(α_distribution, β_distribution, γ_distribution) -Rotate 2D spatial data around the center by an angle chosen at -uniformly from [-γ, γ], an angle given in degrees. +Rotate spatial data around its center. Rotate(γ) is a 2D rotation +by an angle chosen uniformly from [-γ, γ], an angle given in degrees. +Rotate(α, β, γ) is a 3D rotation by angles chosen uniformly from +[-α, α], [-β, β], and [-γ, γ], for X, Y, and Z rotations. You can also pass any `Distributions.Sampleable` from which the angle is selected. @@ -131,25 +135,77 @@ angle is selected. ## Examples ```julia -tfm = Rotate(10) -``` +tfm2d = Rotate(10) +apply(tfm2d, Image(rand(Float32, 16, 16))) +tfm3d = Rotate(10, 20, 30) +apply(tfm3d, Image(rand(Float32, 16, 16, 16))) +``` """ -struct Rotate{S<:Sampleable} <: ProjectiveTransform +struct Rotate{N, R, S<:Sampleable} <: ProjectiveTransform dist::S end -Rotate(γ) = Rotate(Uniform(-abs(γ), abs(γ))) + +Rotate{N, R}(s::S) where {N, R, S} = Rotate{N, R, S}(s) + +function Rotate{N, R}(s::S) where {N, R, S<:Number} + if s == 0 + return Identity() + end + Rotate{N, R, Uniform}(Uniform(-abs(s), abs(s))) +end + +Rotate(s) = Rotate{2, Type{RotMatrix{2, Float64}}}(s) +Rotate(sx, sy, sz) = RotateX(sx) |> RotateY(sy) |> RotateZ(sz) +Rotate{3}(s) = Rotate(s, s, s) +Rotate{2}(s) = Rotate(s) + +""" + RotateX(γ) + RotateX(distribution) + +X-Axis rotation of 3D spatial data around the center by an angle chosen +uniformly from [-γ, γ], an angle given in degrees. + +You can also pass any `Distributions.Sampleable` from which the +angle is selected. +""" +RotateX(s) = Rotate{3, Type{RotX{Float64}}}(s) + +""" + RotateY(γ) + RotateY(distribution) + +Y-Axis rotation of 3D spatial data around the center by an angle chosen +uniformly from [-γ, γ], an angle given in degrees. + +You can also pass any `Distributions.Sampleable` from which the +angle is selected. +""" +RotateY(s) = Rotate{3, Type{RotY{Float64}}}(s) + +""" + RotateZ(γ) + RotateZ(distribution) + +Z-Axis rotation of 3D spatial data around the center by an angle chosen +uniformly from [-γ, γ], an angle given in degrees. + +You can also pass any `Distributions.Sampleable` from which the +angle is selected. +""" +RotateZ(s) = Rotate{3, Type{RotZ{Float64}}}(s) getrandstate(tfm::Rotate) = rand(tfm.dist) function getprojection( - tfm::Rotate, - bounds::Bounds{2}; - randstate = getrandstate(tfm)) + tfm::Rotate{N, Type{R}, S}, + bounds::Bounds{N}; + randstate = getrandstate(tfm)) where {N, R, S} γ = randstate - middlepoint = SVector{2, Float32}(mean.(bounds.rs)) + middlepoint = SVector{N, Float32}(mean.(bounds.rs)) r = γ / 360 * 2pi - return recenter(RotMatrix(convert(Float32, r)), middlepoint) + return recenter(RotMatrix{N, Float32}(R(convert(Float64, r))), middlepoint) end diff --git a/test/projective/affine.jl b/test/projective/affine.jl index a8c0ae09..1d513652 100644 --- a/test/projective/affine.jl +++ b/test/projective/affine.jl @@ -107,15 +107,65 @@ include("../imports.jl") P = DataAugmentation.getprojection(tfm, getbounds(image)) @test P isa AffineMap @test P.linear.mat[1] isa Float32 + timage = apply(tfm, image, randstate=180) + @test itemdata(image) ≈ itemdata(timage)[end:-1:1, end:-1:1] end - @testset ExtendedTestSet "Rotate" begin + @testset ExtendedTestSet "Rotate3D" begin + image = Image(rand(RGB, 10, 20, 30)) + tfm1 = Rotate(180, 180, 180) + tfm2 = Rotate{3}(180) + @test_nowarn apply(tfm1, image) + @test_nowarn apply(tfm2, image) + + # Test equivalent rotations result in the same image. Both rotations + # should invert the x and y axis. + timage1 = apply(tfm1, image, randstate=[180, 180, 0]) + timage2 = apply(tfm2, image, randstate=[0, 0, 180]) + @test image.bounds == timage1.bounds + @test image.bounds == timage2.bounds + @test size(itemdata(image)) == size(itemdata(timage1)) + @test size(itemdata(image)) == size(itemdata(timage2)) + @test itemdata(image) ≈ itemdata(timage1)[end:-1:1, end:-1:1, :] + @test itemdata(image) ≈ itemdata(timage2)[end:-1:1, end:-1:1, :] + end + + @testset ExtendedTestSet "RotateX" begin + tfm = RotateX(180) + image = Image(rand(Float32, 10, 20, 30)) + @test_nowarn apply(tfm, image) + transformed = apply(tfm, image, randstate=180) + @test image.bounds == transformed.bounds + @test size(itemdata(image)) == size(itemdata(transformed)) + @test itemdata(image) ≈ itemdata(transformed)[:, end:-1:1, end:-1:1] + end + + @testset ExtendedTestSet "RotateY" begin + tfm = RotateY(180) + image = Image(rand(Float32, 10, 20, 30)) + @test_nowarn apply(tfm, image) + transformed = apply(tfm, image, randstate=180) + @test image.bounds == transformed.bounds + @test size(itemdata(image)) == size(itemdata(transformed)) + @test itemdata(image) ≈ itemdata(transformed)[end:-1:1, :, end:-1:1] + end + + @testset ExtendedTestSet "RotateZ" begin + tfm = RotateZ(180) + image = Image(rand(Float32, 10, 20, 30)) + @test_nowarn apply(tfm, image) + transformed = apply(tfm, image, randstate=180) + @test image.bounds == transformed.bounds + @test size(itemdata(image)) == size(itemdata(transformed)) + @test itemdata(image) ≈ itemdata(transformed)[end:-1:1, end:-1:1, :] + end + + @testset ExtendedTestSet "Zoom" begin tfm = Zoom((0.1, 2.)) image = Image(rand(RGB, 50, 50)) @test_nowarn apply(tfm, image) end - @testset ExtendedTestSet "Reflect" begin tfm = Reflect(10) testprojective(tfm) @@ -177,6 +227,7 @@ end ) tfms = compose( + Rotate(10, 20, 30), ScaleRatio((.8, .8, .8)), ScaleKeepAspect((12, 10, 10)), RandomCrop((10, 10, 10)) From a6a0f4d539b89be9e3e92b5afa8c15c992c67d12 Mon Sep 17 00:00:00 2001 From: Paul Novotny Date: Sun, 14 Jul 2024 09:56:53 -0400 Subject: [PATCH 2/4] Add check to `getprojection` for `ComposedProjectiveTransform` Ensure the number of transforms equals the number of random states. Previously, the projection would silently be calculated only for the number of transforms equal to the number of random states. --- src/projective/compose.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/projective/compose.jl b/src/projective/compose.jl index 94e01ef8..4562b30e 100644 --- a/src/projective/compose.jl +++ b/src/projective/compose.jl @@ -40,6 +40,7 @@ function getprojection( composed::ComposedProjectiveTransform, bounds; randstate = getrandstate(composed)) + @assert length(composed.tfms) == length(randstate) P = CoordinateTransformations.IdentityTransformation() for (tfm, r) in zip(composed.tfms, randstate) P_tfm = getprojection(tfm, bounds; randstate = r) From 112fb66c63735615a7041cd1892ccb22e149e210 Mon Sep 17 00:00:00 2001 From: Paul Novotny Date: Sat, 20 Jul 2024 11:11:08 -0400 Subject: [PATCH 3/4] Add RotateX, RotateY, and RotateZ to documentation --- docs/src/ref.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/ref.md b/docs/src/ref.md index d2911a73..8f48df72 100644 --- a/docs/src/ref.md +++ b/docs/src/ref.md @@ -10,6 +10,9 @@ RandomResizeCrop CenterResizeCrop Crop Rotate +RotateX +RotateY +RotateZ itemdata showitems From 301319bee6c77d75605dbf1ca0e7d8a1296db71d Mon Sep 17 00:00:00 2001 From: Paul Novotny Date: Sat, 20 Jul 2024 11:18:00 -0400 Subject: [PATCH 4/4] Clean up documentation, add 3D rotation to gallery * Use `@docs; canonical=false` for documentation outside of ref.md. This allows documentation to exist on a second page. Previously, documentation was split between multiple pages. Some were in ref.md, and some were in other pages (transformations.md, buffering.md). Now, all documentation is at least in ref.md, but can also be duplicated on other pages with `canonical=false`. * Add missing documentation to ref.md, and transformations.md. * Enable the visualizations in gallery.md, and clean up the code a bit. * Add visualizations in gallery.md for 3D rotations. --- docs/src/buffering.md | 2 +- docs/src/projective/gallery.md | 98 +++++++++++----------------------- docs/src/ref.md | 20 ++++++- docs/src/transformations.md | 22 +++++--- 4 files changed, 64 insertions(+), 78 deletions(-) diff --git a/docs/src/buffering.md b/docs/src/buffering.md index eb3a8fcd..7be39baa 100644 --- a/docs/src/buffering.md +++ b/docs/src/buffering.md @@ -22,7 +22,7 @@ buffer = apply(tfm, item) # uses apply! internally Since `Buffered` only stores one buffer, you may run into problems when using it in a multi-threading context where different threads invalidate the buffer before it can be used. In that case, you can use [`DataAugmentation.BufferedThreadsafe`](@ref), a version of `Buffered` that keeps a separate buffer for every thread. -```@docs +```@docs; canonical=false DataAugmentation.Buffered DataAugmentation.BufferedThreadsafe ``` diff --git a/docs/src/projective/gallery.md b/docs/src/projective/gallery.md index c4139e6e..2e726ab1 100644 --- a/docs/src/projective/gallery.md +++ b/docs/src/projective/gallery.md @@ -7,15 +7,7 @@ the keypoint-based items [`Keypoints`](@ref), [`Polygon`](@ref), and [`BoundingB Let's take this picture of a light house: -```@setup deps -using DataAugmentation -using MosaicViews -using Images -using TestImages -using StaticArrays -``` - -```@example +```@example deps using DataAugmentation using MosaicViews using Images @@ -26,11 +18,6 @@ imagedata = testimage("lighthouse") imagedata = imresize(imagedata, ratio = 196 / size(imagedata, 1)) ``` -```@example deps -imagedata = testimage("lighthouse") -imagedata -``` - To apply a transformation `tfm` to it, wrap it in `Image`, apply the transformation and unwrap it using [`itemdata`](@ref): @@ -74,59 +61,29 @@ apply(tfm, (image, bbox)) |> showitems Of course, you have to create a 3-dimensional transformation, i.e. `CenterCrop((128, 128, 128))` instead of `CenterCrop((128, 128))`. -## Gallery -```julia -function showtransform(tfm, item, n = 8; ncol = 4) - return mosaicview( - [showitems(apply(tfm, item)) for _ in 1:n], - fillvalue = RGBA(1, 1, 1, 0), - npad = 8, - rowmajor = true, - ncol = ncol) -end - - -function showtransforms(tfms, item; ncol = length(tfms)) - return mosaicview( - [parent(showitems(apply(tfm, item))) for tfm in tfms], - fillvalue = RGBA(1, 1, 1, 0), - npad = 8, - rowmajor = true, - ncol = ncol) -end - -nothing # hide -``` - -### [`RandomResizeCrop`](@ref)`(sz)` +## [`RandomResizeCrop`](@ref)`(sz)` Resizes the sides so that one of them is no longer than `sz` and crops a region of size `sz` *from a random location*. -```julia +```@example deps tfm = RandomResizeCrop((128, 128)) +showgrid([apply(tfm, (image, bbox)) for _ in 1:6]; ncol=6, npad=8) ``` -```julia -o = showtransform(tfm, (image, bbox), 6, ncol=6) -``` - -### [`CenterResizeCrop`](@ref) +## [`CenterResizeCrop`](@ref) Resizes the sides so that one of them is no longer than `sz` and crops a region of size `sz` *from the center*. -```julia +```@example deps tfm = CenterResizeCrop((128, 128)) +showgrid([apply(tfm, (image, bbox))]; ncol=6, npad=8) ``` -```julia -o = showtransform(tfm, (image, bbox), 1) -``` - -### [`Crop`](@ref)`(sz[, from])` +## [`Crop`](@ref)`(sz[, from])` Crops a region of size `sz` from the image, *without resizing* the image first. -```julia +```@example deps using DataAugmentation: FromOrigin, FromCenter, FromRandom tfms = [ Crop((128, 128), FromOrigin()), @@ -136,36 +93,43 @@ tfms = [ Crop((128, 128), FromRandom()), Crop((128, 128), FromRandom()), ] +showgrid([apply(tfm, (image, bbox)) for tfm in tfms]; ncol=6, npad=8) ``` -```julia -o = showtransforms(tfms, (image, bbox)) -``` - -### [`FlipX`](@ref), [`FlipY`](@ref), [`Reflect`](@ref) +## [`FlipX`](@ref), [`FlipY`](@ref), [`Reflect`](@ref) Flip the data on the horizontally and vertically, respectively. More generally, reflect around an angle from the x-axis. -```julia +```@example deps tfms = [ FlipX(), FlipY(), Reflect(30), ] +showgrid([apply(tfm, (image, bbox)) for tfm in tfms]; ncol=6, npad=8) ``` -```julia -o = showtransforms(tfms, (image, bbox)) -``` - -### [`Rotate`](@ref) +## [`Rotate`](@ref), [`RotateX`](@ref), [`RotateY`](@ref), [`RotateZ`](@ref) -Rotate counter-clockwise by an angle. +Rotate a 2D image counter-clockwise by an angle. -```julia +```@example deps tfm = Rotate(20) |> CenterCrop((256, 256)) +showgrid([apply(tfm, (image, bbox)) for _ in 1:6]; ncol=6, npad=8) ``` -```julia -o = showtransform(tfm, (image, bbox), 1) +Rotate also works with 3D images in addition to 3D specific transforms RotateX, RotateY, and RotateZ. + +```@example deps +image3D = Image([RGB(i, j, k) for i=0:0.01:1, j=0:0.01:1, k=0:0.01:1]) +tfms = [ + Rotate(20, 30, 40), + Rotate{3}(45), + RotateX(45), + RotateY(45), + RotateZ(45), +] +transformed = [apply(tfm, image3D) |> itemdata for tfm in tfms] +slices = [Image(parent(t[:, :, 50])) for t in transformed] +showgrid(slices; ncol=6, npad=8) ``` diff --git a/docs/src/ref.md b/docs/src/ref.md index 8f48df72..4924a934 100644 --- a/docs/src/ref.md +++ b/docs/src/ref.md @@ -1,28 +1,44 @@ # Reference ```@docs +AdjustBrightness +AdjustContrast BoundingBox +CenterCrop +CenterResizeCrop +Crop +FlipX +FlipY Image Keypoints MaskBinary MaskMulti +Maybe +OneOf +PermuteDims Polygon +RandomCrop RandomResizeCrop -CenterResizeCrop -Crop +Reflect Rotate RotateX RotateY RotateZ +ScaleKeepAspect +ScaleRatio +WarpAffine itemdata showitems DataAugmentation.AbstractArrayItem DataAugmentation.AbstractItem DataAugmentation.ArrayItem +DataAugmentation.Buffered +DataAugmentation.BufferedThreadsafe DataAugmentation.Categorify DataAugmentation.ComposedProjectiveTransform DataAugmentation.FillMissing DataAugmentation.Identity +DataAugmentation.ImageToTensor DataAugmentation.Item DataAugmentation.ItemWrapper DataAugmentation.MapElem diff --git a/docs/src/transformations.md b/docs/src/transformations.md index 449776df..11a8d3de 100644 --- a/docs/src/transformations.md +++ b/docs/src/transformations.md @@ -25,20 +25,26 @@ Projective transformations include: Affine transformations are a subgroup of projective transformations that can be composed very efficiently: composing two affine transformations results in another affine transformation. Affine transformations can represent translation, scaling, reflection and rotation. Available `Transform`s are: -```@docs -ScaleRatio -ScaleKeepAspect +```@docs; canonical=false FlipX FlipY Reflect +Rotate +RotateX +RotateY +RotateZ +ScaleKeepAspect +ScaleFixed +ScaleRatio WarpAffine +Zoom ``` ## Crops To get a cropped result, simply `compose` any `ProjectiveTransform` with -```@docs +```@docs; canonical=false CenterCrop RandomCrop ``` @@ -47,9 +53,9 @@ RandomCrop DataAugmentation.jl currently supports the following color transformations for augmentation: -```@docs -AdjustContrast +```@docs; canonical=false AdjustBrightness +AdjustContrast ``` # Stochastic transformations @@ -57,7 +63,7 @@ When augmenting data, it is often useful to apply a transformation only with som - [`Maybe`](@ref)`(tfm, p = 0.5)` applies a transformation with probability `p`; and - [`OneOf`](@ref)`([tfm1, tfm2])` randomly selects a transformation to apply. -```@docs +```@docs; canonical=false Maybe OneOf ``` @@ -72,6 +78,6 @@ titems = [apply(tfm, item) for _ in 1:8] showgrid(titems; ncol = 4, npad = 16) ``` -```@docs +```@docs; canonical=false DataAugmentation.ImageToTensor ```