| 1 | package kaka.cakelight; |
| 2 | |
| 3 | import org.opencv.core.Core; |
| 4 | import org.opencv.core.CvType; |
| 5 | import org.opencv.core.Mat; |
| 6 | import org.opencv.core.Size; |
| 7 | import org.opencv.imgproc.Imgproc; |
| 8 | |
| 9 | import static kaka.cakelight.Main.saveFile; |
| 10 | import static kaka.cakelight.Main.timeIt; |
| 11 | |
| 12 | public class VideoFrame { |
| 13 | private byte[] data; |
| 14 | private Configuration config; |
| 15 | // private Mat colImage; |
| 16 | // private Mat rowImage; |
| 17 | private Mat converted; |
| 18 | private Mat[] images; |
| 19 | |
| 20 | private VideoFrame(byte[] data) { |
| 21 | this.data = data; |
| 22 | } |
| 23 | |
| 24 | public static VideoFrame of(byte[] data, Configuration config) { |
| 25 | VideoFrame frame = new VideoFrame(data); |
| 26 | frame.config = config; |
| 27 | if (config.video.mjpg) { |
| 28 | frame.convertJpg(); |
| 29 | } else { |
| 30 | frame.convert(); |
| 31 | } |
| 32 | return frame; |
| 33 | } |
| 34 | |
| 35 | private void convertJpg() { |
| 36 | Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC3); // 8-bit, unsigned, 3 channels |
| 37 | src.put(0, 0, data); |
| 38 | // save(src, "/home/kaka/test-src.data"); |
| 39 | |
| 40 | converted = new Mat(); |
| 41 | Imgproc.cvtColor(src, converted, Imgproc.COLOR_BGR2RGB); |
| 42 | |
| 43 | Core.flip(converted, converted, 0); // up-side down |
| 44 | // save(converted, "/home/kaka/test-converted.data"); |
| 45 | int mysteriousPixelShift = 18; |
| 46 | converted = converted.submat( // crop mysterious pixel shift |
| 47 | 0, |
| 48 | converted.rows(), |
| 49 | mysteriousPixelShift, |
| 50 | converted.cols() - mysteriousPixelShift |
| 51 | ); |
| 52 | // save(converted, "/home/kaka/test-croppedAgain.data"); |
| 53 | model4(converted, Imgproc.INTER_AREA); |
| 54 | src.release(); |
| 55 | converted.release(); |
| 56 | } |
| 57 | |
| 58 | private void convert() { |
| 59 | /* TODO: how to do this? |
| 60 | 1) Resize to an image with the size of the number of leds and use config to define how many pixels deep into the screen to use. |
| 61 | 2) Resize to 16x9 and use 2 pixels of depth (or maybe 3) and interpolate for each led. |
| 62 | 3) Resize to 2 images where each led uses 2 pixels: |
| 63 | vertical - 16 x <#leds> |
| 64 | horizontal - <#leds> x 9 |
| 65 | 4) Resize to cols x rows first, then resize to a vertical and a horizontal like in (3). |
| 66 | */ |
| 67 | Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC2); // 8-bit, unsigned, 2 channels |
| 68 | src.put(0, 0, data); |
| 69 | |
| 70 | // Mat converted = new Mat(); |
| 71 | // Mat resized = new Mat(); |
| 72 | // |
| 73 | // timeIt("total", () -> { |
| 74 | // timeIt("yuyv2rgb", () -> Imgproc.cvtColor(src, converted, Imgproc.COLOR_YUV2RGB_YUYV)); // 3.5 - 4.0 ms |
| 75 | // timeIt("resizing", () -> Imgproc.resize(converted, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, Imgproc.INTER_AREA)); // INTER_AREA is the best for shrinking, but also the slowest (~1.5 ms) |
| 76 | // }); |
| 77 | |
| 78 | Mat cropped = src.submat( |
| 79 | config.video.crop.top, |
| 80 | config.video.height - config.video.crop.bottom, |
| 81 | config.video.crop.left, |
| 82 | config.video.width - config.video.crop.right |
| 83 | ); |
| 84 | converted = new Mat(); |
| 85 | Imgproc.cvtColor(cropped, converted, config.video.format); |
| 86 | // timeIt("model 1", () -> model1(converted, Imgproc.INTER_AREA)); |
| 87 | // timeIt("model 2", () -> model2(converted, Imgproc.INTER_AREA)); |
| 88 | // timeIt("model 3", () -> model3(converted, Imgproc.INTER_AREA)); |
| 89 | timeIt("model 4", () -> model4(converted, Imgproc.INTER_AREA)); |
| 90 | // save(converted, "/home/kaka/test-converted.data"); |
| 91 | // save(resized, "/home/kaka/test-resized.data"); |
| 92 | src.release(); |
| 93 | cropped.release(); |
| 94 | } |
| 95 | |
| 96 | private void model1(Mat src, int interpolation) { |
| 97 | Mat resized = new Mat(); |
| 98 | Imgproc.resize(src, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, interpolation); |
| 99 | } |
| 100 | |
| 101 | private void model2(Mat src, int interpolation) { |
| 102 | Mat resized = new Mat(); |
| 103 | Imgproc.resize(src, resized, new Size(16, 9), 0, 0, interpolation); |
| 104 | } |
| 105 | |
| 106 | private void model3(Mat src, int interpolation) { |
| 107 | // colImage = new Mat(); |
| 108 | // rowImage = new Mat(); |
| 109 | // Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation); |
| 110 | // Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation); |
| 111 | } |
| 112 | |
| 113 | private void model4(Mat src, int interpolation) { |
| 114 | int width = 3 * src.cols() / 16; |
| 115 | int height = 3 * src.rows() / 9; |
| 116 | Mat[] cropped = new Mat[] { |
| 117 | /* LEFT */ src.submat(0, src.rows(), 0, width), |
| 118 | /* RIGHT */ src.submat(0, src.rows(), src.cols() - width, src.cols()), |
| 119 | /* TOP */ src.submat(0, height, 0, src.cols()), |
| 120 | /* BOTTOM */ src.submat(src.rows() - height, src.rows(), 0, src.cols()), |
| 121 | }; |
| 122 | images = new Mat[] {new Mat(), new Mat(), new Mat(), new Mat()}; |
| 123 | // Imgproc.resize(cropped[ListPosition.LEFT.ordinal()], images[ListPosition.LEFT.ordinal()], new Size(3, config.leds.rows), 0, 0, interpolation); |
| 124 | Imgproc.resize(cropped[0], images[0], new Size(3, config.leds.rows), 0, 0, interpolation); |
| 125 | Imgproc.resize(cropped[1], images[1], new Size(3, config.leds.rows), 0, 0, interpolation); |
| 126 | Imgproc.resize(cropped[2], images[2], new Size(config.leds.cols, 3), 0, 0, interpolation); |
| 127 | Imgproc.resize(cropped[3], images[3], new Size(config.leds.cols, 3), 0, 0, interpolation); |
| 128 | // Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation); |
| 129 | // Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation); |
| 130 | } |
| 131 | |
| 132 | private Color wrappedGetLedColor(ListPosition listPosition, int xy) { |
| 133 | Color c = getLedColor(listPosition, xy); |
| 134 | double[] hsv = c.toHSV(); |
| 135 | double saturation = config.video.saturation >= 0.5 |
| 136 | ? hsv[1] + (config.video.saturation - 0.5) * 2 * (1 - hsv[1]) |
| 137 | : hsv[1] - (1 - config.video.saturation * 2) * hsv[1]; |
| 138 | return Color.hsv(hsv[0], saturation, hsv[2]); |
| 139 | } |
| 140 | |
| 141 | private Color getLedColor(ListPosition listPosition, int xy) { |
| 142 | // TODO: maybe use highest value from pixels? 100 % from 1st, 66 % from 2nd, 33 % from 3rd. colors might be strange. |
| 143 | switch (listPosition) { |
| 144 | case LEFT: |
| 145 | return interpolatedRowColor(images[0], xy, 0, 1, 2); |
| 146 | // return interpolatedRowColor(xy, 0, 1, 2); |
| 147 | case RIGHT: |
| 148 | return interpolatedRowColor(images[1], xy, 2, 1, 0); |
| 149 | // return interpolatedRowColor(xy, 15, 14, 13); |
| 150 | case TOP: |
| 151 | return interpolatedColColor(images[2], xy, 0, 1, 2); |
| 152 | // return interpolatedColColor(xy, 0, 1, 2); |
| 153 | case BOTTOM: |
| 154 | return interpolatedColColor(images[3], xy, 2, 1, 0); |
| 155 | // return interpolatedColColor(xy, 8, 7, 6); |
| 156 | } |
| 157 | return null; |
| 158 | } |
| 159 | |
| 160 | // private Color interpolatedRowColor(int y, int x1, int x2, int x3) { |
| 161 | private Color interpolatedRowColor(Mat rowImage, int y, int x1, int x2, int x3) { |
| 162 | return pixelToColor(rowImage, x3, y).interpolate(pixelToColor(rowImage, x2, y), 0.65).interpolate(pixelToColor(rowImage, x1, y), 0.65); |
| 163 | } |
| 164 | |
| 165 | // private Color interpolatedColColor(int x, int y1, int y2, int y3) { |
| 166 | private Color interpolatedColColor(Mat colImage, int x, int y1, int y2, int y3) { |
| 167 | return pixelToColor(colImage, x, y3).interpolate(pixelToColor(colImage, x, y2), 0.65).interpolate(pixelToColor(colImage, x, y1), 0.65); |
| 168 | } |
| 169 | |
| 170 | private Color pixelToColor(Mat image, int x, int y) { |
| 171 | byte[] rgb = new byte[3]; |
| 172 | image.get(y, x, rgb); |
| 173 | return Color.rgb(rgb[0] & 0xff, rgb[1] & 0xff, rgb[2] & 0xff); |
| 174 | } |
| 175 | |
| 176 | private void save(Mat mat, String filepath) { |
| 177 | byte[] data = new byte[mat.cols() * mat.rows() * mat.channels()]; |
| 178 | mat.get(0, 0, data); |
| 179 | saveFile(data, filepath); |
| 180 | } |
| 181 | |
| 182 | public byte[] getData() { |
| 183 | byte[] buff = new byte[(int) (converted.total() * converted.channels())]; |
| 184 | converted.get(0, 0, buff); |
| 185 | return buff; |
| 186 | } |
| 187 | |
| 188 | // public Mat getColImage() { |
| 189 | // return colImage; |
| 190 | // } |
| 191 | |
| 192 | // public Mat getRowImage() { |
| 193 | // return rowImage; |
| 194 | // } |
| 195 | |
| 196 | public Mat getConvertedImage() { |
| 197 | return converted; |
| 198 | } |
| 199 | |
| 200 | /** |
| 201 | * Creates a LED frame going counter-clockwise from the bottom-left corner, sans the corners. |
| 202 | */ |
| 203 | public LedFrame getLedFrame() { |
| 204 | LedFrame frame = LedFrame.from(config); |
| 205 | int led = 0; |
| 206 | |
| 207 | if (config.video.list.bottom) |
| 208 | for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.BOTTOM, i)); |
| 209 | else |
| 210 | for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, Color.BLACK); |
| 211 | |
| 212 | if (config.video.list.right) |
| 213 | for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.RIGHT, i)); |
| 214 | else |
| 215 | for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK); |
| 216 | |
| 217 | if (config.video.list.top) |
| 218 | for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.TOP, i)); |
| 219 | else |
| 220 | for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK); |
| 221 | |
| 222 | if (config.video.list.left) |
| 223 | for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.LEFT, i)); |
| 224 | else |
| 225 | for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, Color.BLACK); |
| 226 | |
| 227 | return frame; |
| 228 | } |
| 229 | |
| 230 | private enum ListPosition { |
| 231 | LEFT, |
| 232 | RIGHT, |
| 233 | TOP, |
| 234 | BOTTOM |
| 235 | } |
| 236 | } |