Commit | Line | Data |
---|---|---|
e59e98fc TW |
1 | package kaka.cakelight; |
2 | ||
eba8feca | 3 | import org.opencv.core.Core; |
e59e98fc TW |
4 | import org.opencv.core.CvType; |
5 | import org.opencv.core.Mat; | |
6 | import org.opencv.core.Size; | |
7 | import org.opencv.imgproc.Imgproc; | |
8 | ||
9 | import static kaka.cakelight.Main.saveFile; | |
10 | import static kaka.cakelight.Main.timeIt; | |
11 | ||
adc29b9a | 12 | public class VideoFrame { |
e59e98fc TW |
13 | private byte[] data; |
14 | private Configuration config; | |
14d53f06 TW |
15 | // private Mat colImage; |
16 | // private Mat rowImage; | |
100b82fe | 17 | private Mat converted; |
14d53f06 | 18 | private Mat[] images; |
e59e98fc | 19 | |
adc29b9a | 20 | private VideoFrame(byte[] data) { |
e59e98fc TW |
21 | this.data = data; |
22 | } | |
23 | ||
adc29b9a TW |
24 | public static VideoFrame of(byte[] data, Configuration config) { |
25 | VideoFrame frame = new VideoFrame(data); | |
e59e98fc | 26 | frame.config = config; |
eba8feca TW |
27 | if (config.video.mjpg) { |
28 | frame.convertJpg(); | |
29 | } else { | |
30 | frame.convert(); | |
31 | } | |
e59e98fc TW |
32 | return frame; |
33 | } | |
34 | ||
eba8feca TW |
35 | private void convertJpg() { |
36 | Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC3); // 8-bit, unsigned, 3 channels | |
37 | src.put(0, 0, data); | |
38 | // save(src, "/home/kaka/test-src.data"); | |
39 | ||
40 | converted = new Mat(); | |
41 | Imgproc.cvtColor(src, converted, Imgproc.COLOR_BGR2RGB); | |
42 | ||
43 | Core.flip(converted, converted, 0); // up-side down | |
44 | // save(converted, "/home/kaka/test-converted.data"); | |
45 | int mysteriousPixelShift = 18; | |
46 | converted = converted.submat( // crop mysterious pixel shift | |
47 | 0, | |
48 | converted.rows(), | |
49 | mysteriousPixelShift, | |
50 | converted.cols() - mysteriousPixelShift | |
51 | ); | |
52 | // save(converted, "/home/kaka/test-croppedAgain.data"); | |
53 | model4(converted, Imgproc.INTER_AREA); | |
54 | src.release(); | |
55 | converted.release(); | |
56 | } | |
57 | ||
e59e98fc | 58 | private void convert() { |
4a2d6056 TW |
59 | /* TODO: how to do this? |
60 | 1) Resize to an image with the size of the number of leds and use config to define how many pixels deep into the screen to use. | |
61 | 2) Resize to 16x9 and use 2 pixels of depth (or maybe 3) and interpolate for each led. | |
62 | 3) Resize to 2 images where each led uses 2 pixels: | |
63 | vertical - 16 x <#leds> | |
64 | horizontal - <#leds> x 9 | |
cc03403a | 65 | 4) Resize to cols x rows first, then resize to a vertical and a horizontal like in (3). |
4a2d6056 | 66 | */ |
e59e98fc TW |
67 | Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC2); // 8-bit, unsigned, 2 channels |
68 | src.put(0, 0, data); | |
69 | ||
4a2d6056 TW |
70 | // Mat converted = new Mat(); |
71 | // Mat resized = new Mat(); | |
72 | // | |
73 | // timeIt("total", () -> { | |
74 | // timeIt("yuyv2rgb", () -> Imgproc.cvtColor(src, converted, Imgproc.COLOR_YUV2RGB_YUYV)); // 3.5 - 4.0 ms | |
75 | // timeIt("resizing", () -> Imgproc.resize(converted, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, Imgproc.INTER_AREA)); // INTER_AREA is the best for shrinking, but also the slowest (~1.5 ms) | |
76 | // }); | |
e59e98fc | 77 | |
100b82fe TW |
78 | Mat cropped = src.submat( |
79 | config.video.crop.top, | |
80 | config.video.height - config.video.crop.bottom, | |
81 | config.video.crop.left, | |
82 | config.video.width - config.video.crop.right | |
83 | ); | |
84 | converted = new Mat(); | |
cc03403a | 85 | Imgproc.cvtColor(cropped, converted, config.video.format); |
100b82fe TW |
86 | // timeIt("model 1", () -> model1(converted, Imgproc.INTER_AREA)); |
87 | // timeIt("model 2", () -> model2(converted, Imgproc.INTER_AREA)); | |
14d53f06 TW |
88 | // timeIt("model 3", () -> model3(converted, Imgproc.INTER_AREA)); |
89 | timeIt("model 4", () -> model4(converted, Imgproc.INTER_AREA)); | |
e59e98fc TW |
90 | // save(converted, "/home/kaka/test-converted.data"); |
91 | // save(resized, "/home/kaka/test-resized.data"); | |
100b82fe TW |
92 | src.release(); |
93 | cropped.release(); | |
4a2d6056 TW |
94 | } |
95 | ||
96 | private void model1(Mat src, int interpolation) { | |
97 | Mat resized = new Mat(); | |
98 | Imgproc.resize(src, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, interpolation); | |
99 | } | |
100 | ||
101 | private void model2(Mat src, int interpolation) { | |
102 | Mat resized = new Mat(); | |
103 | Imgproc.resize(src, resized, new Size(16, 9), 0, 0, interpolation); | |
104 | } | |
105 | ||
106 | private void model3(Mat src, int interpolation) { | |
14d53f06 TW |
107 | // colImage = new Mat(); |
108 | // rowImage = new Mat(); | |
109 | // Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation); | |
110 | // Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation); | |
111 | } | |
112 | ||
113 | private void model4(Mat src, int interpolation) { | |
114 | int width = 3 * src.cols() / 16; | |
115 | int height = 3 * src.rows() / 9; | |
116 | Mat[] cropped = new Mat[] { | |
117 | /* LEFT */ src.submat(0, src.rows(), 0, width), | |
118 | /* RIGHT */ src.submat(0, src.rows(), src.cols() - width, src.cols()), | |
119 | /* TOP */ src.submat(0, height, 0, src.cols()), | |
120 | /* BOTTOM */ src.submat(src.rows() - height, src.rows(), 0, src.cols()), | |
121 | }; | |
122 | images = new Mat[] {new Mat(), new Mat(), new Mat(), new Mat()}; | |
123 | // Imgproc.resize(cropped[ListPosition.LEFT.ordinal()], images[ListPosition.LEFT.ordinal()], new Size(3, config.leds.rows), 0, 0, interpolation); | |
124 | Imgproc.resize(cropped[0], images[0], new Size(3, config.leds.rows), 0, 0, interpolation); | |
125 | Imgproc.resize(cropped[1], images[1], new Size(3, config.leds.rows), 0, 0, interpolation); | |
126 | Imgproc.resize(cropped[2], images[2], new Size(config.leds.cols, 3), 0, 0, interpolation); | |
127 | Imgproc.resize(cropped[3], images[3], new Size(config.leds.cols, 3), 0, 0, interpolation); | |
128 | // Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation); | |
129 | // Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation); | |
130 | } | |
131 | ||
b3e10312 TW |
132 | private Color wrappedGetLedColor(ListPosition listPosition, int xy) { |
133 | Color c = getLedColor(listPosition, xy); | |
134 | double[] hsv = c.toHSV(); | |
a80ebf3e TW |
135 | double saturation = config.video.saturation >= 0.5 |
136 | ? hsv[1] + (config.video.saturation - 0.5) * 2 * (1 - hsv[1]) | |
137 | : hsv[1] - (1 - config.video.saturation * 2) * hsv[1]; | |
138 | return Color.hsv(hsv[0], saturation, hsv[2]); | |
b3e10312 TW |
139 | } |
140 | ||
14d53f06 | 141 | private Color getLedColor(ListPosition listPosition, int xy) { |
cc03403a | 142 | // TODO: maybe use highest value from pixels? 100 % from 1st, 66 % from 2nd, 33 % from 3rd. colors might be strange. |
4a2d6056 TW |
143 | switch (listPosition) { |
144 | case LEFT: | |
14d53f06 TW |
145 | return interpolatedRowColor(images[0], xy, 0, 1, 2); |
146 | // return interpolatedRowColor(xy, 0, 1, 2); | |
4a2d6056 | 147 | case RIGHT: |
14d53f06 TW |
148 | return interpolatedRowColor(images[1], xy, 2, 1, 0); |
149 | // return interpolatedRowColor(xy, 15, 14, 13); | |
4a2d6056 | 150 | case TOP: |
14d53f06 TW |
151 | return interpolatedColColor(images[2], xy, 0, 1, 2); |
152 | // return interpolatedColColor(xy, 0, 1, 2); | |
4a2d6056 | 153 | case BOTTOM: |
14d53f06 TW |
154 | return interpolatedColColor(images[3], xy, 2, 1, 0); |
155 | // return interpolatedColColor(xy, 8, 7, 6); | |
4a2d6056 TW |
156 | } |
157 | return null; | |
158 | } | |
159 | ||
14d53f06 TW |
160 | // private Color interpolatedRowColor(int y, int x1, int x2, int x3) { |
161 | private Color interpolatedRowColor(Mat rowImage, int y, int x1, int x2, int x3) { | |
100b82fe TW |
162 | return pixelToColor(rowImage, x3, y).interpolate(pixelToColor(rowImage, x2, y), 0.65).interpolate(pixelToColor(rowImage, x1, y), 0.65); |
163 | } | |
164 | ||
14d53f06 TW |
165 | // private Color interpolatedColColor(int x, int y1, int y2, int y3) { |
166 | private Color interpolatedColColor(Mat colImage, int x, int y1, int y2, int y3) { | |
100b82fe TW |
167 | return pixelToColor(colImage, x, y3).interpolate(pixelToColor(colImage, x, y2), 0.65).interpolate(pixelToColor(colImage, x, y1), 0.65); |
168 | } | |
169 | ||
4a2d6056 TW |
170 | private Color pixelToColor(Mat image, int x, int y) { |
171 | byte[] rgb = new byte[3]; | |
172 | image.get(y, x, rgb); | |
100b82fe | 173 | return Color.rgb(rgb[0] & 0xff, rgb[1] & 0xff, rgb[2] & 0xff); |
e59e98fc TW |
174 | } |
175 | ||
176 | private void save(Mat mat, String filepath) { | |
177 | byte[] data = new byte[mat.cols() * mat.rows() * mat.channels()]; | |
178 | mat.get(0, 0, data); | |
179 | saveFile(data, filepath); | |
180 | } | |
181 | ||
182 | public byte[] getData() { | |
100b82fe TW |
183 | byte[] buff = new byte[(int) (converted.total() * converted.channels())]; |
184 | converted.get(0, 0, buff); | |
185 | return buff; | |
186 | } | |
187 | ||
14d53f06 TW |
188 | // public Mat getColImage() { |
189 | // return colImage; | |
190 | // } | |
100b82fe | 191 | |
14d53f06 TW |
192 | // public Mat getRowImage() { |
193 | // return rowImage; | |
194 | // } | |
100b82fe TW |
195 | |
196 | public Mat getConvertedImage() { | |
197 | return converted; | |
e59e98fc | 198 | } |
03b67a73 TW |
199 | |
200 | /** | |
da7bef43 | 201 | * Creates a LED frame going counter-clockwise from the bottom-left corner, sans the corners. |
03b67a73 TW |
202 | */ |
203 | public LedFrame getLedFrame() { | |
204 | LedFrame frame = LedFrame.from(config); | |
205 | int led = 0; | |
a4fb845a TW |
206 | |
207 | if (config.video.list.bottom) | |
208 | for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.BOTTOM, i)); | |
209 | else | |
210 | for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, Color.BLACK); | |
211 | ||
212 | if (config.video.list.right) | |
213 | for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.RIGHT, i)); | |
214 | else | |
215 | for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK); | |
216 | ||
217 | if (config.video.list.top) | |
218 | for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.TOP, i)); | |
219 | else | |
220 | for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK); | |
221 | ||
222 | if (config.video.list.left) | |
223 | for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.LEFT, i)); | |
224 | else | |
225 | for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, Color.BLACK); | |
226 | ||
03b67a73 TW |
227 | return frame; |
228 | } | |
48cb60b7 TW |
229 | |
230 | private enum ListPosition { | |
231 | LEFT, | |
232 | RIGHT, | |
233 | TOP, | |
234 | BOTTOM | |
235 | } | |
e59e98fc | 236 | } |