From 0db10ff8c2c31f70407bded2eb89b275d08cdff0 Mon Sep 17 00:00:00 2001 From: Dexton Anderson Date: Fri, 1 Nov 2024 10:29:28 -0500 Subject: [PATCH 1/2] Revert "Fix some bullshit (#21)" This reverts commit 80872925b2cb4601099298e8a0e26c283f2e55c6. --- build.gradle.kts | 14 +- .../java/org/bytedeco/javacv/CanvasFrame.java | 449 +++++ .../bytedeco/javacv/FFmpegFrameGrabber.java | 1604 +++++++++++++++++ .../bytedeco/javacv/FFmpegFrameRecorder.java | 1442 +++++++++++++++ src/main/java/org/bytedeco/javacv/Frame.java | 380 ++++ .../org/bytedeco/javacv/FrameConverter.java | 49 + .../org/bytedeco/javacv/FrameGrabber.java | 810 +++++++++ .../org/bytedeco/javacv/FrameRecorder.java | 442 +++++ .../bytedeco/javacv/Java2DFrameConverter.java | 739 ++++++++ .../bytedeco/javacv/OpenCVFrameConverter.java | 251 +++ .../bytedeco/javacv/OpenCVFrameGrabber.java | 318 ++++ .../java/org/bytedeco/javacv/Seekable.java | 28 + .../logger/MagewellVideoDataLogger.java | 4 +- 13 files changed, 6525 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/bytedeco/javacv/CanvasFrame.java create mode 100644 src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java create mode 100644 src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java create mode 100644 src/main/java/org/bytedeco/javacv/Frame.java create mode 100644 src/main/java/org/bytedeco/javacv/FrameConverter.java create mode 100644 src/main/java/org/bytedeco/javacv/FrameGrabber.java create mode 100644 src/main/java/org/bytedeco/javacv/FrameRecorder.java create mode 100644 src/main/java/org/bytedeco/javacv/Java2DFrameConverter.java create mode 100644 src/main/java/org/bytedeco/javacv/OpenCVFrameConverter.java create mode 100644 src/main/java/org/bytedeco/javacv/OpenCVFrameGrabber.java create mode 100644 src/main/java/org/bytedeco/javacv/Seekable.java diff --git a/build.gradle.kts b/build.gradle.kts index 3bbd0651..7d21eba2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -49,9 +49,17 @@ mainDependencies { api("com.hierynomus:sshj:0.31.0") api("us.ihmc:scs2-definition:17-0.28.1") - api("org.bytedeco:javacv:1.5.9") - api("org.bytedeco:javacpp:1.5.9") - api("org.bytedeco:javacv-platform:1.5.9") + val opencvVersion = "4.7.0-1.5.9" + api("org.bytedeco:opencv:$opencvVersion") + api("org.bytedeco:opencv:$opencvVersion:linux-x86_64") + api("org.bytedeco:opencv:$opencvVersion:linux-arm64") + api("org.bytedeco:opencv:$opencvVersion:windows-x86_64") + val ffmpegVersion = "6.0-1.5.9" + api("org.bytedeco:ffmpeg:$ffmpegVersion") + api("org.bytedeco:ffmpeg:$ffmpegVersion:linux-x86_64") + api("org.bytedeco:ffmpeg:$ffmpegVersion:linux-arm64") + api("org.bytedeco:ffmpeg:$ffmpegVersion:windows-x86_64") + api("org.freedesktop.gstreamer:gst1-java-core:1.4.0") var javaFXVersion = "17.0.2" diff --git a/src/main/java/org/bytedeco/javacv/CanvasFrame.java b/src/main/java/org/bytedeco/javacv/CanvasFrame.java new file mode 100644 index 00000000..f36c1be6 --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/CanvasFrame.java @@ -0,0 +1,449 @@ +/* + * Copyright (C) 2009-2015 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +import java.awt.Canvas; +import java.awt.Color; +import java.awt.Component; +import java.awt.Dimension; +import java.awt.DisplayMode; +import java.awt.EventQueue; +import java.awt.Graphics; +import java.awt.Graphics2D; +import java.awt.GraphicsConfiguration; +import java.awt.GraphicsDevice; +import java.awt.GraphicsEnvironment; +import java.awt.Image; +import java.awt.KeyEventDispatcher; +import java.awt.KeyboardFocusManager; +import java.awt.Transparency; +import java.awt.color.ColorSpace; +import java.awt.color.ICC_ColorSpace; +import java.awt.color.ICC_ProfileRGB; +import java.awt.event.ComponentAdapter; +import java.awt.event.ComponentEvent; +import java.awt.event.KeyEvent; +import java.awt.image.BufferStrategy; +import java.awt.image.BufferedImage; +import javax.swing.JFrame; +import javax.swing.JRootPane; + +/** + * + * @author Samuel Audet + * + * Make sure OpenGL or XRender is enabled to get low latency, something like + * export _JAVA_OPTIONS=-Dsun.java2d.opengl=True + * export _JAVA_OPTIONS=-Dsun.java2d.xrender=True + */ +public class CanvasFrame extends JFrame { + + public static class Exception extends java.lang.Exception { + public Exception(String message) { super(message); } + public Exception(String message, Throwable cause) { super(message, cause); } + } + + public static String[] getScreenDescriptions() { + GraphicsDevice[] screens = getScreenDevices(); + String[] descriptions = new String[screens.length]; + for (int i = 0; i < screens.length; i++) { + descriptions[i] = screens[i].getIDstring(); + } + return descriptions; + } + public static DisplayMode getDisplayMode(int screenNumber) { + GraphicsDevice[] screens = getScreenDevices(); + if (screenNumber >= 0 && screenNumber < screens.length) { + return screens[screenNumber].getDisplayMode(); + } else { + return null; + } + } + public static double getGamma(int screenNumber) { + GraphicsDevice[] screens = getScreenDevices(); + if (screenNumber >= 0 && screenNumber < screens.length) { + return getGamma(screens[screenNumber]); + } else { + return 0.0; + } + } + public static double getDefaultGamma() { + return getGamma(getDefaultScreenDevice()); + } + + public static double getGamma(GraphicsDevice screen) { + ColorSpace cs = screen.getDefaultConfiguration().getColorModel().getColorSpace(); + if (cs.isCS_sRGB()) { + return 2.2; + } else { + try { + return ((ICC_ProfileRGB)((ICC_ColorSpace)cs).getProfile()).getGamma(0); + } catch (RuntimeException e) { } + } + return 0.0; + } + public static GraphicsDevice getScreenDevice(int screenNumber) throws Exception { + GraphicsDevice[] screens = getScreenDevices(); + if (screenNumber >= screens.length) { + throw new Exception("CanvasFrame Error: Screen number " + screenNumber + " not found. " + + "There are only " + screens.length + " screens."); + } + return screens[screenNumber];//.getDefaultConfiguration(); + } + public static GraphicsDevice[] getScreenDevices() { + return GraphicsEnvironment.getLocalGraphicsEnvironment().getScreenDevices(); + } + public static GraphicsDevice getDefaultScreenDevice() { + return GraphicsEnvironment.getLocalGraphicsEnvironment().getDefaultScreenDevice(); + } + + public CanvasFrame(String title) { + this(title, 0.0); + } + public CanvasFrame(String title, double gamma) { + super(title); + init(false, null, gamma); + } + + public CanvasFrame(String title, GraphicsConfiguration gc) { + this(title, gc, 0.0); + } + public CanvasFrame(String title, GraphicsConfiguration gc, double gamma) { + super(title, gc); + init(false, null, gamma); + } + + public CanvasFrame(String title, int screenNumber, DisplayMode displayMode) throws Exception { + this(title, screenNumber, displayMode, 0.0); + } + public CanvasFrame(String title, int screenNumber, DisplayMode displayMode, double gamma) throws Exception { + super(title, getScreenDevice(screenNumber).getDefaultConfiguration()); + init(true, displayMode, gamma); + } + + private void init(final boolean fullScreen, final DisplayMode displayMode, final double gamma) { + Runnable r = new Runnable() { public void run() { + KeyboardFocusManager.getCurrentKeyboardFocusManager(). + addKeyEventDispatcher(keyEventDispatch); + + GraphicsDevice gd = getGraphicsConfiguration().getDevice(); + DisplayMode d = gd.getDisplayMode(), d2 = null; + if (displayMode != null && d != null) { + int w = displayMode.getWidth(); + int h = displayMode.getHeight(); + int b = displayMode.getBitDepth(); + int r = displayMode.getRefreshRate(); + d2 = new DisplayMode(w > 0 ? w : d.getWidth(), h > 0 ? h : d.getHeight(), + b > 0 ? b : d.getBitDepth(), r > 0 ? r : d.getRefreshRate()); + } + if (fullScreen) { + setUndecorated(true); + getRootPane().setWindowDecorationStyle(JRootPane.NONE); + setResizable(false); + gd.setFullScreenWindow(CanvasFrame.this); + } else { + setLocationByPlatform(true); + } + if (d2 != null && !d2.equals(d)) { + gd.setDisplayMode(d2); + } + double g = gamma == 0.0 ? getGamma(gd) : gamma; + inverseGamma = g == 0.0 ? 1.0 : 1.0/g; + + // Must be called after the fullscreen stuff, but before + // getting our BufferStrategy or even creating our Canvas + setVisible(true); + + initCanvas(fullScreen, displayMode, gamma); + }}; + + if (EventQueue.isDispatchThread()) { + r.run(); + } else { + try { + EventQueue.invokeAndWait(r); + } catch (java.lang.Exception ex) { } + } + } + + protected void initCanvas(boolean fullScreen, DisplayMode displayMode, double gamma) { + canvas = new Canvas() { + @Override public void update(Graphics g) { + paint(g); + } + @Override public void paint(Graphics g) { + // Calling BufferStrategy.show() here sometimes throws + // NullPointerException or IllegalStateException, + // but otherwise seems to work fine. + try { + if (canvas.getWidth() <= 0 || canvas.getHeight() <= 0) { + return; + } + BufferStrategy strategy = canvas.getBufferStrategy(); + do { + do { + g = strategy.getDrawGraphics(); + if (color != null) { + g.setColor(color); + g.fillRect(0, 0, getWidth(), getHeight()); + } + if (image != null) { + g.drawImage(image, 0, 0, getWidth(), getHeight(), null); + } + if (buffer != null) { + g.drawImage(buffer, 0, 0, getWidth(), getHeight(), null); + } + g.dispose(); + } while (strategy.contentsRestored()); + strategy.show(); + } while (strategy.contentsLost()); + } catch (NullPointerException e) { + } catch (IllegalStateException e) { } + } + }; + if (fullScreen) { + canvas.setSize(getSize()); + needInitialResize = false; + } else { + canvas.setSize(10,10); // mac bug + needInitialResize = true; + } + getContentPane().add(canvas); + canvas.setVisible(true); + canvas.createBufferStrategy(2); + //canvas.setIgnoreRepaint(true); + } + + // used for example as debugging console... + public static CanvasFrame global = null; + + // Latency is about 60 ms on Metacity and Windows XP, and 90 ms on Compiz Fusion, + // but we set the default to twice as much to take into account the roundtrip + // camera latency as well, just to be sure + public static final long DEFAULT_LATENCY = 200; + private long latency = DEFAULT_LATENCY; + + private KeyEvent keyEvent = null; + private KeyEventDispatcher keyEventDispatch = new KeyEventDispatcher() { + public boolean dispatchKeyEvent(KeyEvent e) { + if (e.getID() == KeyEvent.KEY_PRESSED) { + synchronized (CanvasFrame.this) { + keyEvent = e; + CanvasFrame.this.notify(); + } + } + return false; + } + }; + + protected Canvas canvas = null; + protected boolean needInitialResize = false; + protected double initialScale = 1.0; + protected double inverseGamma = 1.0; + private Color color = null; + private Image image = null; + private BufferedImage buffer = null; + private Java2DFrameConverter converter = new Java2DFrameConverter(); + + public long getLatency() { + // if there exists some way to estimate the latency in real time, + // add it here + return latency; + } + public void setLatency(long latency) { + this.latency = latency; + } + public void waitLatency() throws InterruptedException { + Thread.sleep(getLatency()); + } + + public KeyEvent waitKey() throws InterruptedException { + return waitKey(0); + } + public synchronized KeyEvent waitKey(int delay) throws InterruptedException { + if (delay >= 0) { + keyEvent = null; + wait(delay); + } + KeyEvent e = keyEvent; + keyEvent = null; + return e; + } + + public Canvas getCanvas() { + return canvas; + } + + public Dimension getCanvasSize() { + return canvas.getSize(); + } + public void setCanvasSize(final int width, final int height) { + Dimension d = getCanvasSize(); + if (d.width == width && d.height == height) { + return; + } + + Runnable r = new Runnable() { public void run() { + // There is apparently a bug in Java code for Linux, and what happens goes like this: + // 1. Canvas gets resized, checks the visible area (has not changed) and updates + // BufferStrategy with the same size. 2. pack() resizes the frame and changes + // the visible area 3. We call Canvas.setSize() with different dimensions, to make + // it check the visible area and reallocate the BufferStrategy almost correctly + // 4. Finally, we resize the Canvas to the desired size... phew! + setExtendedState(NORMAL); // force unmaximization + canvas.setSize(width, height); + pack(); + canvas.setSize(width+1, height+1); + canvas.setSize(width, height); + needInitialResize = false; + }}; + + if (EventQueue.isDispatchThread()) { + r.run(); + } else { + try { + EventQueue.invokeAndWait(r); + } catch (java.lang.Exception ex) { } + } + } + + public double getCanvasScale() { + return initialScale; + } + public void setCanvasScale(double initialScale) { + this.initialScale = initialScale; + this.needInitialResize = true; + } + + public Graphics2D createGraphics() { + if (buffer == null || buffer.getWidth() != canvas.getWidth() || buffer.getHeight() != canvas.getHeight()) { + BufferedImage newbuffer = canvas.getGraphicsConfiguration().createCompatibleImage( + canvas.getWidth(), canvas.getHeight(), Transparency.TRANSLUCENT); + if (buffer != null) { + Graphics g = newbuffer.getGraphics(); + g.drawImage(buffer, 0, 0, null); + g.dispose(); + } + buffer = newbuffer; + } + return buffer.createGraphics(); + } + public void releaseGraphics(Graphics2D g) { + g.dispose(); + canvas.paint(null); + } + + public void showColor(Color color) { + this.color = color; + this.image = null; + canvas.paint(null); + } + + // Java2D will do gamma correction for TYPE_CUSTOM BufferedImage, but + // not for the standard types, so we need to do it manually. + public void showImage(Frame image) { + showImage(image, false); + } + public void showImage(Frame image, boolean flipChannels) { + showImage(converter.getBufferedImage(image, converter.getBufferedImageType(image) == + BufferedImage.TYPE_CUSTOM ? 1.0 : inverseGamma, flipChannels, null)); + } + public void showImage(Image image) { + if (image == null) { + return; + } else if (isResizable() && needInitialResize) { + int w = (int)Math.round(image.getWidth (null)*initialScale); + int h = (int)Math.round(image.getHeight(null)*initialScale); + setCanvasSize(w, h); + } + this.color = null; + this.image = image; + canvas.paint(null); + } + + // This should not be called from the event dispatch thread (EDT), + // but if it is, it should not totally crash... In the worst case, + // it will simply timeout waiting for the moved events. + public static void tile(final CanvasFrame[] frames) { + + class MovedListener extends ComponentAdapter { + boolean moved = false; + @Override public void componentMoved(ComponentEvent e) { + moved = true; + Component c = e.getComponent(); + synchronized (c) { + c.notify(); + } + } + } + final MovedListener movedListener = new MovedListener(); + + // layout the canvas frames for the cameras in tiles + int canvasCols = (int)Math.round(Math.sqrt(frames.length)); + if (canvasCols*canvasCols < frames.length) { + // if we don't get a square, favor horizontal layouts + // since screens are usually wider than cameras... + // and we also have title bars, tasks bar, menus, etc that + // takes up vertical space + canvasCols++; + } + int canvasX = 0, canvasY = 0; + int canvasMaxY = 0; + for (int i = 0; i < frames.length; i++) { + final int n = i; + final int x = canvasX; + final int y = canvasY; + try { + movedListener.moved = false; + EventQueue.invokeLater(new Runnable() { + public void run() { + frames[n].addComponentListener(movedListener); + frames[n].setLocation(x, y); + } + }); + int count = 0; + while (!movedListener.moved && count < 5) { + // wait until the window manager actually places our window... + // wait a maximum of 500 ms since this does not work if + // we are on the event dispatch thread. also some window + // managers like Windows do not always send us the event... + synchronized (frames[n]) { + frames[n].wait(100); + } + count++; + } + EventQueue.invokeLater(new Runnable() { + public void run() { + frames[n].removeComponentListener(movedListener); + } + }); + } catch (java.lang.Exception ex) { } + canvasX = frames[i].getX()+frames[i].getWidth(); + canvasMaxY = Math.max(canvasMaxY, frames[i].getY()+frames[i].getHeight()); + if ((i+1)%canvasCols == 0) { + canvasX = 0; + canvasY = canvasMaxY; + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java new file mode 100644 index 00000000..8c7acfa6 --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java @@ -0,0 +1,1604 @@ +/* + * Copyright (C) 2009-2024 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + * Based on the avcodec_sample.0.5.0.c file available at + * http://web.me.com/dhoerl/Home/Tech_Blog/Entries/2009/1/22_Revised_avcodec_sample.c_files/avcodec_sample.0.5.0.c + * by Martin Böhme, Stephen Dranger, and David Hoerl + * as well as on the decoding_encoding.c file included in FFmpeg 0.11.1, + * and on the decode_video.c file included in FFmpeg 4.4, + * which is covered by the following copyright notice: + * + * Copyright (c) 2001 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.bytedeco.javacv; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerScope; +import org.bytedeco.javacpp.PointerPointer; + +import org.bytedeco.ffmpeg.avcodec.*; +import org.bytedeco.ffmpeg.avformat.*; +import org.bytedeco.ffmpeg.avutil.*; +import org.bytedeco.ffmpeg.swresample.*; +import org.bytedeco.ffmpeg.swscale.*; +import static org.bytedeco.ffmpeg.global.avcodec.*; +import static org.bytedeco.ffmpeg.global.avdevice.*; +import static org.bytedeco.ffmpeg.global.avformat.*; +import static org.bytedeco.ffmpeg.global.avutil.*; +import static org.bytedeco.ffmpeg.global.swresample.*; +import static org.bytedeco.ffmpeg.global.swscale.*; + +/** + * + * @author Samuel Audet + */ +public class FFmpegFrameGrabber extends FrameGrabber { + + public static class Exception extends FrameGrabber.Exception { + public Exception(String message) { super(message + " (For more details, make sure FFmpegLogCallback.set() has been called.)"); } + public Exception(String message, Throwable cause) { super(message, cause); } + } + + public static String[] getDeviceDescriptions() throws Exception { + tryLoad(); + throw new UnsupportedOperationException("Device enumeration not support by FFmpeg."); + } + + public static FFmpegFrameGrabber createDefault(File deviceFile) throws Exception { return new FFmpegFrameGrabber(deviceFile); } + public static FFmpegFrameGrabber createDefault(String devicePath) throws Exception { return new FFmpegFrameGrabber(devicePath); } + public static FFmpegFrameGrabber createDefault(int deviceNumber) throws Exception { throw new Exception(FFmpegFrameGrabber.class + " does not support device numbers."); } + + private static Exception loadingException = null; + public static void tryLoad() throws Exception { + if (loadingException != null) { + throw loadingException; + } else { + try { + Loader.load(org.bytedeco.ffmpeg.global.avutil.class); + Loader.load(org.bytedeco.ffmpeg.global.swresample.class); + Loader.load(org.bytedeco.ffmpeg.global.avcodec.class); + Loader.load(org.bytedeco.ffmpeg.global.avformat.class); + Loader.load(org.bytedeco.ffmpeg.global.swscale.class); + + // Register all formats and codecs + av_jni_set_java_vm(Loader.getJavaVM(), null); + // avcodec_register_all(); + // av_register_all(); + avformat_network_init(); + + Loader.load(org.bytedeco.ffmpeg.global.avdevice.class); + avdevice_register_all(); + } catch (Throwable t) { + if (t instanceof Exception) { + throw loadingException = (Exception)t; + } else { + throw loadingException = new Exception("Failed to load " + FFmpegFrameGrabber.class, t); + } + } + } + } + + static { + try { + tryLoad(); + // FFmpegLockCallback.init(); + } catch (Exception ex) { } + } + + public FFmpegFrameGrabber(URL url) { + this(url.toString()); + } + public FFmpegFrameGrabber(File file) { + this(file.getAbsolutePath()); + } + public FFmpegFrameGrabber(String filename) { + this.filename = filename; + this.pixelFormat = AV_PIX_FMT_NONE; + this.sampleFormat = AV_SAMPLE_FMT_NONE; + } + /** Calls {@code FFmpegFrameGrabber(inputStream, Integer.MAX_VALUE - 8)} + * so that the whole input stream is seekable. */ + public FFmpegFrameGrabber(InputStream inputStream) { + this(inputStream, Integer.MAX_VALUE - 8); + } + /** Set maximumSize to 0 to disable seek and minimize startup time. */ + public FFmpegFrameGrabber(InputStream inputStream, int maximumSize) { + this.inputStream = inputStream; + this.closeInputStream = true; + this.pixelFormat = AV_PIX_FMT_NONE; + this.sampleFormat = AV_SAMPLE_FMT_NONE; + this.maximumSize = maximumSize; + } + public void release() throws Exception { + synchronized (org.bytedeco.ffmpeg.global.avcodec.class) { + releaseUnsafe(); + } + } + public synchronized void releaseUnsafe() throws Exception { + started = false; + + if (plane_ptr != null && plane_ptr2 != null) { + plane_ptr.releaseReference(); + plane_ptr2.releaseReference(); + plane_ptr = plane_ptr2 = null; + } + + if (pkt != null) { + if (pkt.stream_index() != -1) { + av_packet_unref(pkt); + } + pkt.releaseReference(); + pkt = null; + } + + if (default_layout != null) { + default_layout.releaseReference(); + default_layout = null; + } + + // Free the RGB image + if (image_ptr != null) { + for (int i = 0; i < image_ptr.length; i++) { + if (imageMode != ImageMode.RAW) { + av_free(image_ptr[i]); + } + } + image_ptr = null; + } + if (picture_rgb != null) { + av_frame_free(picture_rgb); + picture_rgb = null; + } + + // Free the native format picture frame + if (picture != null) { + av_frame_free(picture); + picture = null; + } + + // Close the video codec + if (video_c != null) { + avcodec_free_context(video_c); + video_c = null; + } + + // Free the audio samples frame + if (samples_frame != null) { + av_frame_free(samples_frame); + samples_frame = null; + } + + // Close the audio codec + if (audio_c != null) { + avcodec_free_context(audio_c); + audio_c = null; + } + + // Close the video file + if (inputStream == null && oc != null && !oc.isNull()) { + avformat_close_input(oc); + oc = null; + } + + if (img_convert_ctx != null) { + sws_freeContext(img_convert_ctx); + img_convert_ctx = null; + } + + if (samples_ptr_out != null) { + for (int i = 0; i < samples_ptr_out.length; i++) { + av_free(samples_ptr_out[i].position(0)); + } + samples_ptr_out = null; + samples_buf_out = null; + } + + if (samples_convert_ctx != null) { + swr_free(samples_convert_ctx); + samples_convert_ctx.releaseReference(); + samples_convert_ctx = null; + } + + frameGrabbed = false; + frame = null; + timestamp = 0; + frameNumber = 0; + + if (inputStream != null) { + try { + if (oc == null) { + // when called a second time + if (closeInputStream) { + inputStream.close(); + } + } else if (maximumSize > 0) { + try { + inputStream.reset(); + } catch (IOException ex) { + // "Resetting to invalid mark", give up? + System.err.println("Error on InputStream.reset(): " + ex); + } + } + } catch (IOException ex) { + throw new Exception("Error on InputStream.close(): ", ex); + } finally { + inputStreams.remove(oc); + if (avio != null) { + if (avio.buffer() != null) { + av_free(avio.buffer()); + avio.buffer(null); + } + av_free(avio); + avio = null; + } + if (oc != null) { + avformat_close_input(oc); + oc = null; + } + } + } + } + @Override protected void finalize() throws Throwable { + super.finalize(); + release(); + } + + static Map inputStreams = Collections.synchronizedMap(new HashMap()); + + static class ReadCallback extends Read_packet_Pointer_BytePointer_int { + @Override public int call(Pointer opaque, BytePointer buf, int buf_size) { + try { + byte[] b = new byte[buf_size]; + InputStream is = inputStreams.get(opaque); + int size = is.read(b, 0, buf_size); + if (size < 0) { + return AVERROR_EOF(); + } else { + buf.put(b, 0, size); + return size; + } + } + catch (Throwable t) { + System.err.println("Error on InputStream.read(): " + t); + return -1; + } + } + } + + static class SeekCallback extends Seek_Pointer_long_int { + @Override public long call(Pointer opaque, long offset, int whence) { + try { + InputStream is = inputStreams.get(opaque); + long size = 0; + switch (whence) { + case 0: is.reset(); break; // SEEK_SET + case 1: break; // SEEK_CUR + case 2: // SEEK_END + is.reset(); + while (true) { + long n = is.skip(Long.MAX_VALUE); + if (n == 0) break; + size += n; + } + offset += size; + is.reset(); + break; + case AVSEEK_SIZE: + long remaining = 0; + while (true) { + long n = is.skip(Long.MAX_VALUE); + if (n == 0) break; + remaining += n; + } + is.reset(); + while (true) { + long n = is.skip(Long.MAX_VALUE); + if (n == 0) break; + size += n; + } + offset = size - remaining; + is.reset(); + break; + default: return -1; + } + long remaining = offset; + while (remaining > 0) { + long skipped = is.skip(remaining); + if (skipped == 0) break; // end of the stream + remaining -= skipped; + } + return whence == AVSEEK_SIZE ? size : 0; + } catch (Throwable t) { + System.err.println("Error on InputStream.reset() or skip(): " + t); + return -1; + } + } + } + + static ReadCallback readCallback = new ReadCallback().retainReference(); + static SeekCallback seekCallback = new SeekCallback().retainReference(); + + private InputStream inputStream; + private boolean closeInputStream; + private int maximumSize; + private AVIOContext avio; + private String filename; + private AVFormatContext oc; + private AVStream video_st, audio_st; + private AVCodecContext video_c, audio_c; + private AVFrame picture, picture_rgb; + private BytePointer[] image_ptr; + private Buffer[] image_buf; + private AVFrame samples_frame; + private BytePointer[] samples_ptr; + private Buffer[] samples_buf; + private BytePointer[] samples_ptr_out; + private Buffer[] samples_buf_out; + private PointerPointer plane_ptr, plane_ptr2; + private AVPacket pkt; + private SwsContext img_convert_ctx; + private SwrContext samples_convert_ctx; + private int samples_channels, samples_format, samples_rate; + private boolean frameGrabbed; + private Frame frame; + private int[] streams; + private AVChannelLayout default_layout; + + private volatile boolean started = false; + + public boolean isCloseInputStream() { + return closeInputStream; + } + public void setCloseInputStream(boolean closeInputStream) { + this.closeInputStream = closeInputStream; + } + + /** + * Is there a video stream? + * @return {@code video_st!=null;} + */ + public boolean hasVideo() { + return video_st!=null; + } + + /** + * Is there an audio stream? + * @return {@code audio_st!=null;} + */ + public boolean hasAudio() { + return audio_st!=null; + } + + @Override public double getGamma() { + // default to a gamma of 2.2 for cheap Webcams, DV cameras, etc. + if (gamma == 0.0) { + return 2.2; + } else { + return gamma; + } + } + + @Override public String getFormat() { + if (oc == null) { + return super.getFormat(); + } else { + return oc.iformat().name().getString(); + } + } + + @Override public int getImageWidth() { + return imageWidth > 0 || video_c == null ? super.getImageWidth() : video_c.width(); + } + + @Override public int getImageHeight() { + return imageHeight > 0 || video_c == null ? super.getImageHeight() : video_c.height(); + } + + @Override public int getAudioChannels() { + return audioChannels > 0 || audio_c == null ? super.getAudioChannels() : audio_c.ch_layout().nb_channels(); + } + + @Override public int getPixelFormat() { + if (imageMode == ImageMode.COLOR || imageMode == ImageMode.GRAY) { + if (pixelFormat == AV_PIX_FMT_NONE) { + return imageMode == ImageMode.COLOR ? AV_PIX_FMT_BGR24 : AV_PIX_FMT_GRAY8; + } else { + return pixelFormat; + } + } else if (video_c != null) { // RAW + return video_c.pix_fmt(); + } else { + return super.getPixelFormat(); + } + } + + @Override public int getVideoCodec() { + return video_c == null ? super.getVideoCodec() : video_c.codec_id(); + } + + @Override + public String getVideoCodecName(){ + return video_c == null ? super.getVideoCodecName() : video_c.codec().name().getString(); + } + + @Override public int getVideoBitrate() { + return video_c == null ? super.getVideoBitrate() : (int)video_c.bit_rate(); + } + + @Override public double getAspectRatio() { + if (video_st == null) { + return super.getAspectRatio(); + } else { + AVRational r = av_guess_sample_aspect_ratio(oc, video_st, picture); + double a = (double)r.num() / r.den(); + return a == 0.0 ? 1.0 : a; + } + } + + /** Returns {@link #getVideoFrameRate()} */ + @Override public double getFrameRate() { + return getVideoFrameRate(); + } + + /**Estimation of audio frames per second. + * + * Care must be taken as this method may require unnecessary call of + * grabFrame(true, false, false, false, false) with frameGrabbed set to true. + * + * @return (double) getSampleRate()) / samples_frame.nb_samples() + * if samples_frame.nb_samples() is not zero, otherwise return 0 + */ + public double getAudioFrameRate() { + if (audio_st == null) { + return 0.0; + } else { + if (samples_frame == null || samples_frame.nb_samples() == 0) { + try { + grabFrame(true, false, false, false, false); + frameGrabbed = true; + } catch (Exception e) { + return 0.0; + } + } + if (samples_frame != null && samples_frame.nb_samples() != 0) + return ((double) getSampleRate()) / samples_frame.nb_samples(); + else return 0.0; + + } + } + + public double getVideoFrameRate() { + if (video_st == null) { + return super.getFrameRate(); + } else { + AVRational r = video_st.avg_frame_rate(); + if (r.num() == 0 && r.den() == 0) { + r = video_st.r_frame_rate(); + } + return (double)r.num() / r.den(); + } + } + + @Override public int getAudioCodec() { + return audio_c == null ? super.getAudioCodec() : audio_c.codec_id(); + } + + @Override public String getAudioCodecName() { + return audio_c == null ? super.getAudioCodecName() : audio_c.codec().name().getString(); + } + + @Override public int getAudioBitrate() { + return audio_c == null ? super.getAudioBitrate() : (int)audio_c.bit_rate(); + } + + @Override public int getSampleFormat() { + if (sampleMode == SampleMode.SHORT || sampleMode == SampleMode.FLOAT) { + if (sampleFormat == AV_SAMPLE_FMT_NONE) { + return sampleMode == SampleMode.SHORT ? AV_SAMPLE_FMT_S16 : AV_SAMPLE_FMT_FLT; + } else { + return sampleFormat; + } + } else if (audio_c != null) { // RAW + return audio_c.sample_fmt(); + } else { + return super.getSampleFormat(); + } + } + + @Override public int getSampleRate() { + return sampleRate > 0 || audio_c == null ? super.getSampleRate() : audio_c.sample_rate(); + } + + @Override public Map getMetadata() { + if (oc == null) { + return super.getMetadata(); + } + AVDictionaryEntry entry = null; + Map metadata = new HashMap(); + while ((entry = av_dict_get(oc.metadata(), "", entry, AV_DICT_IGNORE_SUFFIX)) != null) { + metadata.put(entry.key().getString(charset), entry.value().getString(charset)); + } + return metadata; + } + + @Override public Map getVideoMetadata() { + if (video_st == null) { + return super.getVideoMetadata(); + } + AVDictionaryEntry entry = null; + Map metadata = new HashMap(); + while ((entry = av_dict_get(video_st.metadata(), "", entry, AV_DICT_IGNORE_SUFFIX)) != null) { + metadata.put(entry.key().getString(charset), entry.value().getString(charset)); + } + return metadata; + } + + @Override public Map getAudioMetadata() { + if (audio_st == null) { + return super.getAudioMetadata(); + } + AVDictionaryEntry entry = null; + Map metadata = new HashMap(); + while ((entry = av_dict_get(audio_st.metadata(), "", entry, AV_DICT_IGNORE_SUFFIX)) != null) { + metadata.put(entry.key().getString(charset), entry.value().getString(charset)); + } + return metadata; + } + + @Override public String getMetadata(String key) { + if (oc == null) { + return super.getMetadata(key); + } + AVDictionaryEntry entry = av_dict_get(oc.metadata(), key, null, 0); + return entry == null || entry.value() == null ? null : entry.value().getString(charset); + } + + @Override public String getVideoMetadata(String key) { + if (video_st == null) { + return super.getVideoMetadata(key); + } + AVDictionaryEntry entry = av_dict_get(video_st.metadata(), key, null, 0); + return entry == null || entry.value() == null ? null : entry.value().getString(charset); + } + + @Override public String getAudioMetadata(String key) { + if (audio_st == null) { + return super.getAudioMetadata(key); + } + AVDictionaryEntry entry = av_dict_get(audio_st.metadata(), key, null, 0); + return entry == null || entry.value() == null ? null : entry.value().getString(charset); + } + + @Override public Map getVideoSideData() { + if (video_st == null) { + return super.getVideoSideData(); + } + videoSideData = new HashMap(); + for (int i = 0; i < video_st.nb_side_data(); i++) { + AVPacketSideData sd = video_st.side_data().position(i); + String key = av_packet_side_data_name(sd.type()).getString(); + Buffer value = sd.data().capacity(sd.size()).asBuffer(); + videoSideData.put(key, value); + } + return videoSideData; + } + + @Override public Buffer getVideoSideData(String key) { + return getVideoSideData().get(key); + } + + /** Returns the rotation in degrees from the side data of the video stream, or 0 if unknown. */ + public double getDisplayRotation() { + ByteBuffer b = (ByteBuffer)getVideoSideData("Display Matrix"); + return b != null ? av_display_rotation_get(new IntPointer(new BytePointer(b))) : 0; + } + + @Override public Map getAudioSideData() { + if (audio_st == null) { + return super.getAudioSideData(); + } + audioSideData = new HashMap(); + for (int i = 0; i < audio_st.nb_side_data(); i++) { + AVPacketSideData sd = audio_st.side_data().position(i); + String key = av_packet_side_data_name(sd.type()).getString(); + Buffer value = sd.data().capacity(sd.size()).asBuffer(); + audioSideData.put(key, value); + } + return audioSideData; + } + + @Override public Buffer getAudioSideData(String key) { + return getAudioSideData().get(key); + } + + /** default override of super.setFrameNumber implies setting + * of a frame close to a video frame having that number */ + @Override public void setFrameNumber(int frameNumber) throws Exception { + if (hasVideo()) setTimestamp(Math.round((1000000L * frameNumber + 500000L)/ getFrameRate())); + else super.frameNumber = frameNumber; + } + + /** if there is video stream tries to seek to video frame with corresponding timestamp + * otherwise sets super.frameNumber only because frameRate==0 if there is no video stream */ + public void setVideoFrameNumber(int frameNumber) throws Exception { + // best guess, AVSEEK_FLAG_FRAME has not been implemented in FFmpeg... + if (hasVideo()) setVideoTimestamp(Math.round((1000000L * frameNumber + 500000L)/ getFrameRate())); + else super.frameNumber = frameNumber; + } + + /** if there is audio stream tries to seek to audio frame with corresponding timestamp + * ignoring otherwise */ + public void setAudioFrameNumber(int frameNumber) throws Exception { + // best guess, AVSEEK_FLAG_FRAME has not been implemented in FFmpeg... + if (hasAudio()) setAudioTimestamp(Math.round((1000000L * frameNumber + 500000L)/ getAudioFrameRate())); + } + + /** setTimestamp without checking frame content (using old code used in JavaCV versions prior to 1.4.1) */ + @Override public void setTimestamp(long timestamp) throws Exception { + setTimestamp(timestamp, false); + } + + /** setTimestamp with possibility to select between old quick seek code or new code + * doing check of frame content. The frame check can be useful with corrupted files, when seeking may + * end up with an empty frame not containing video nor audio */ + public void setTimestamp(long timestamp, boolean checkFrame) throws Exception { + setTimestamp(timestamp, checkFrame ? EnumSet.of(Frame.Type.VIDEO, Frame.Type.AUDIO) : null); + } + + /** setTimestamp with resulting video frame type if there is a video stream. + * This should provide precise seek to a video frame containing the requested timestamp + * in most cases. + * */ + public void setVideoTimestamp(long timestamp) throws Exception { + setTimestamp(timestamp, EnumSet.of(Frame.Type.VIDEO)); + } + + /** setTimestamp with resulting audio frame type if there is an audio stream. + * This should provide precise seek to an audio frame containing the requested timestamp + * in most cases. + * */ + public void setAudioTimestamp(long timestamp) throws Exception { + setTimestamp(timestamp, EnumSet.of(Frame.Type.AUDIO)); + } + + /** setTimestamp with a priority the resulting frame should be: + * video (frameTypesToSeek contains only Frame.Type.VIDEO), + * audio (frameTypesToSeek contains only Frame.Type.AUDIO), + * or any (frameTypesToSeek contains both) + */ + private synchronized void setTimestamp(long timestamp, EnumSet frameTypesToSeek) throws Exception { + int ret; + if (oc == null) { + super.timestamp = timestamp; + } else { + timestamp = timestamp * AV_TIME_BASE / 1000000L; + + /* the stream start time */ + long ts0 = oc.start_time() != AV_NOPTS_VALUE ? oc.start_time() : 0; + + if (frameTypesToSeek != null //new code providing check of frame content while seeking to the timestamp + && (frameTypesToSeek.contains(Frame.Type.VIDEO) || frameTypesToSeek.contains(Frame.Type.AUDIO)) + && (hasVideo() || hasAudio())) { + + /* After the call of ffmpeg's avformat_seek_file(...) with the flag set to AVSEEK_FLAG_BACKWARD + * the decoding position should be located before the requested timestamp in a closest position + * from which all the active streams can be decoded successfully. + * The following seeking consists of two stages: + * 1. Grab frames till the frame corresponding to that "closest" position + * (the first frame containing decoded data). + * + * 2. Grab frames till the desired timestamp is reached. The number of steps is restricted + * by doubled estimation of frames between that "closest" position and the desired position. + * + * frameTypesToSeek parameter sets the preferred type of frames to seek. + * It can be chosen from three possible types: VIDEO, AUDIO or any of them. + * The setting means only a preference in the type. That is, if VIDEO or AUDIO is + * specified but the file does not have video or audio stream - any type will be used instead. + */ + + /* Check if file contains requested streams */ + if ((frameTypesToSeek.contains(Frame.Type.VIDEO) && !hasVideo() ) || + (frameTypesToSeek.contains(Frame.Type.AUDIO) && !hasAudio() )) + frameTypesToSeek = EnumSet.of(Frame.Type.VIDEO, Frame.Type.AUDIO); + + /* If frameTypesToSeek is set explicitly to VIDEO or AUDIO + * we need to use start time of the corresponding stream + * instead of the common start time + */ + if (frameTypesToSeek.size()==1) { + if (frameTypesToSeek.contains(Frame.Type.VIDEO)) { + if (video_st!=null && video_st.start_time() != AV_NOPTS_VALUE) { + AVRational time_base = video_st.time_base(); + ts0 = 1000000L * video_st.start_time() * time_base.num() / time_base.den(); + } + } + else if (frameTypesToSeek.contains(Frame.Type.AUDIO)) { + if (audio_st!=null && audio_st.start_time() != AV_NOPTS_VALUE) { + AVRational time_base = audio_st.time_base(); + ts0 = 1000000L * audio_st.start_time() * time_base.num() / time_base.den(); + } + } + } + + /* Sometimes the ffmpeg's avformat_seek_file(...) function brings us not to a position before + * the desired but few frames after. In case we need a frame-precision seek we may + * try to request an earlier timestamp. + */ + long early_ts = timestamp; + + /* add the stream start time */ + timestamp += ts0; + early_ts += ts0; + + long initialSeekPosition = Long.MIN_VALUE; + long maxSeekSteps = 0; + long count = 0; + Frame seekFrame = null; + do { + if ((ret = avformat_seek_file(oc, -1, 0L, early_ts, early_ts, AVSEEK_FLAG_BACKWARD)) < 0) + throw new Exception("avformat_seek_file() error " + ret + ": Could not seek file to timestamp " + timestamp + "."); + if (video_c != null) { + avcodec_flush_buffers(video_c); + } + if (audio_c != null) { + avcodec_flush_buffers(audio_c); + } + if (pkt.stream_index() != -1) { + av_packet_unref(pkt); + pkt.stream_index(-1); + } + seekFrame = grabFrame(frameTypesToSeek.contains(Frame.Type.AUDIO), frameTypesToSeek.contains(Frame.Type.VIDEO), false, false, false); + if (seekFrame == null) return; + initialSeekPosition = seekFrame.timestamp; + if(early_ts==0L) break; + early_ts-=500000L; + if(early_ts<0) early_ts=0L; + } while (initialSeekPosition>timestamp); + double frameDuration = 0.0; + if (seekFrame.image != null && this.getFrameRate() > 0) + frameDuration = AV_TIME_BASE / (double)getFrameRate(); + else if (seekFrame.samples != null && samples_frame != null && getSampleRate() > 0) { + frameDuration = AV_TIME_BASE * samples_frame.nb_samples() / (double)getSampleRate(); + } + + if(frameDuration>0.0) { + maxSeekSteps = 0; //no more grab if the distance to the requested timestamp is smaller than frameDuration + if (timestamp - initialSeekPosition + 1 > frameDuration) //allow for a rounding error + maxSeekSteps = (long)(10*(timestamp - initialSeekPosition)/frameDuration); + } + else if (initialSeekPosition < timestamp) maxSeekSteps = 1000; + + double delta = 0.0; //for the timestamp correction + count = 0; + while(count < maxSeekSteps) { + seekFrame = grabFrame(frameTypesToSeek.contains(Frame.Type.AUDIO), frameTypesToSeek.contains(Frame.Type.VIDEO), false, false, false); + if (seekFrame == null) return; //is it better to throw NullPointerException? + + count++; + double ts=seekFrame.timestamp; + frameDuration = 0.0; + if (seekFrame.image != null && this.getFrameRate() > 0) + frameDuration = AV_TIME_BASE / (double)getFrameRate(); + else if (seekFrame.samples != null && samples_frame != null && getSampleRate() > 0) + frameDuration = AV_TIME_BASE * samples_frame.nb_samples() / (double)getSampleRate(); + + delta = 0.0; + if (frameDuration>0.0) { + delta = (ts-ts0)/frameDuration - Math.round((ts-ts0)/frameDuration); + if (Math.abs(delta)>0.2) delta=0.0; + } + ts-=delta*frameDuration; // corrected timestamp + if (ts + frameDuration > timestamp) break; + } + } else { //old quick seeking code used in JavaCV versions prior to 1.4.1 + /* add the stream start time */ + timestamp += ts0; + if ((ret = avformat_seek_file(oc, -1, Long.MIN_VALUE, timestamp, Long.MAX_VALUE, AVSEEK_FLAG_BACKWARD)) < 0) { + throw new Exception("avformat_seek_file() error " + ret + ": Could not seek file to timestamp " + timestamp + "."); + } + if (video_c != null) { + avcodec_flush_buffers(video_c); + } + if (audio_c != null) { + avcodec_flush_buffers(audio_c); + } + if (pkt.stream_index() != -1) { + av_packet_unref(pkt); + pkt.stream_index(-1); + } + /* comparing to timestamp +/- 1 avoids rouding issues for framerates + which are no proper divisors of 1000000, e.g. where + av_frame_get_best_effort_timestamp in grabFrame sets this.timestamp + to ...666 and the given timestamp has been rounded to ...667 + (or vice versa) + */ + int count = 0; // prevent infinite loops with corrupted files + while (this.timestamp > timestamp + 1 && grabFrame(true, true, false, false) != null && count++ < 1000) { + // flush frames if seeking backwards + } + count = 0; + while (this.timestamp < timestamp - 1 && grabFrame(true, true, false, false) != null && count++ < 1000) { + // decode up to the desired frame + } + } + frameGrabbed = true; + } + } + + /** Returns {@link #getLengthInVideoFrames()} */ + @Override public int getLengthInFrames() { + // best guess... + return getLengthInVideoFrames(); + } + + @Override public long getLengthInTime() { + return oc.duration() * 1000000L / AV_TIME_BASE; + } + + /** Returns {@code (int) Math.round(getLengthInTime() * getFrameRate() / 1000000L)}, which is an approximation in general. */ + public int getLengthInVideoFrames() { + // best guess... + return (int) Math.round(getLengthInTime() * getFrameRate() / 1000000L); + } + + public int getLengthInAudioFrames() { + // best guess... + double afr = getAudioFrameRate(); + if (afr > 0) return (int) (getLengthInTime() * afr / 1000000L); + else return 0; + } + + public AVFormatContext getFormatContext() { + return oc; + } + + /** Calls {@code start(true)}. */ + @Override public void start() throws Exception { + start(true); + } + /** Set findStreamInfo to false to minimize startup time, at the expense of robustness. */ + public void start(boolean findStreamInfo) throws Exception { + synchronized (org.bytedeco.ffmpeg.global.avcodec.class) { + startUnsafe(findStreamInfo); + } + } + public void startUnsafe() throws Exception { + startUnsafe(true); + } + public synchronized void startUnsafe(boolean findStreamInfo) throws Exception { + try (PointerScope scope = new PointerScope()) { + + if (oc != null && !oc.isNull()) { + throw new Exception("start() has already been called: Call stop() before calling start() again."); + } + + int ret; + img_convert_ctx = null; + oc = new AVFormatContext(null); + video_c = null; + audio_c = null; + plane_ptr = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS).retainReference(); + plane_ptr2 = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS).retainReference(); + pkt = new AVPacket().retainReference(); + frameGrabbed = false; + frame = new Frame(); + timestamp = 0; + frameNumber = 0; + default_layout = new AVChannelLayout().retainReference(); + + pkt.stream_index(-1); + + // Open video file + AVInputFormat f = null; + if (format != null && format.length() > 0) { + if ((f = av_find_input_format(format)) == null) { + throw new Exception("av_find_input_format() error: Could not find input format \"" + format + "\"."); + } + } + AVDictionary options = new AVDictionary(null); + if (frameRate > 0) { + AVRational r = av_d2q(frameRate, 1001000); + av_dict_set(options, "framerate", r.num() + "/" + r.den(), 0); + } + if (pixelFormat >= 0) { + av_dict_set(options, "pixel_format", av_get_pix_fmt_name(pixelFormat).getString(), 0); + } else if (imageMode != ImageMode.RAW) { + av_dict_set(options, "pixel_format", imageMode == ImageMode.COLOR ? "bgr24" : "gray8", 0); + } + if (imageWidth > 0 && imageHeight > 0) { + av_dict_set(options, "video_size", imageWidth + "x" + imageHeight, 0); + } + if (sampleRate > 0) { + av_dict_set(options, "sample_rate", "" + sampleRate, 0); + } + if (audioChannels > 0) { + av_dict_set(options, "channels", "" + audioChannels, 0); + } + for (Entry e : this.options.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + if (inputStream != null) { + if (!inputStream.markSupported()) { + inputStream = new BufferedInputStream(inputStream); + } + inputStream.mark(maximumSize); + oc = avformat_alloc_context(); + avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 0, oc, readCallback, null, maximumSize > 0 ? seekCallback : null); + oc.pb(avio); + + filename = inputStream.toString(); + inputStreams.put(oc, inputStream); + } + if ((ret = avformat_open_input(oc, filename, f, options)) < 0) { + av_dict_set(options, "pixel_format", null, 0); + if ((ret = avformat_open_input(oc, filename, f, options)) < 0) { + throw new Exception("avformat_open_input() error " + ret + ": Could not open input \"" + filename + "\". (Has setFormat() been called?)"); + } + } + av_dict_free(options); + + oc.max_delay(maxDelay); + + // Retrieve stream information, if desired + if (findStreamInfo && (ret = avformat_find_stream_info(oc, (PointerPointer)null)) < 0) { + throw new Exception("avformat_find_stream_info() error " + ret + ": Could not find stream information."); + } + + if (av_log_get_level() >= AV_LOG_INFO) { + // Dump information about file onto standard error + av_dump_format(oc, 0, filename, 0); + } + + // Find the first stream with the user-specified disposition property + int nb_streams = oc.nb_streams(); + for (int i = 0; i < nb_streams; i++) { + AVStream st = oc.streams(i); + AVCodecParameters par = st.codecpar(); + if (videoStream < 0 && par.codec_type() == AVMEDIA_TYPE_VIDEO && st.disposition() == videoDisposition) { + videoStream = i; + } else if (audioStream < 0 && par.codec_type() == AVMEDIA_TYPE_AUDIO && st.disposition() == audioDisposition) { + audioStream = i; + } + } + + // Find the first video and audio stream, unless the user specified otherwise + video_st = audio_st = null; + AVCodecParameters video_par = null, audio_par = null; + streams = new int[nb_streams]; + for (int i = 0; i < nb_streams; i++) { + AVStream st = oc.streams(i); + // Get a pointer to the codec context for the video or audio stream + AVCodecParameters par = st.codecpar(); + streams[i] = par.codec_type(); + if (video_st == null && par.codec_type() == AVMEDIA_TYPE_VIDEO && (videoStream < 0 || videoStream == i)) { + video_st = st; + video_par = par; + videoStream = i; + } else if (audio_st == null && par.codec_type() == AVMEDIA_TYPE_AUDIO && (audioStream < 0 || audioStream == i)) { + audio_st = st; + audio_par = par; + audioStream = i; + } + } + if (video_st == null && audio_st == null) { + throw new Exception("Did not find a video or audio stream inside \"" + filename + + "\" for videoStream == " + videoStream + " and audioStream == " + audioStream + "."); + } + + if (video_st != null) { + // Find the decoder for the video stream + AVCodec codec = avcodec_find_decoder_by_name(videoCodecName); + if (codec == null) { + codec = avcodec_find_decoder(video_par.codec_id()); + } + if (codec == null) { + throw new Exception("avcodec_find_decoder() error: Unsupported video format or codec not found: " + video_par.codec_id() + "."); + } + + /* Allocate a codec context for the decoder */ + if ((video_c = avcodec_alloc_context3(codec)) == null) { + throw new Exception("avcodec_alloc_context3() error: Could not allocate video decoding context."); + } + + /* copy the stream parameters from the muxer */ + if ((ret = avcodec_parameters_to_context(video_c, video_st.codecpar())) < 0) { + releaseUnsafe(); + throw new Exception("avcodec_parameters_to_context() error " + ret + ": Could not copy the video stream parameters."); + } + + options = new AVDictionary(null); + for (Entry e : videoOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + + // Enable multithreading when available + video_c.thread_count(0); + + // Open video codec + if ((ret = avcodec_open2(video_c, codec, options)) < 0) { + throw new Exception("avcodec_open2() error " + ret + ": Could not open video codec."); + } + av_dict_free(options); + + // Hack to correct wrong frame rates that seem to be generated by some codecs + if (video_c.time_base().num() > 1000 && video_c.time_base().den() == 1) { + video_c.time_base().den(1000); + } + + // Allocate video frame and an AVFrame structure for the RGB image + if ((picture = av_frame_alloc()) == null) { + throw new Exception("av_frame_alloc() error: Could not allocate raw picture frame."); + } + if ((picture_rgb = av_frame_alloc()) == null) { + throw new Exception("av_frame_alloc() error: Could not allocate RGB picture frame."); + } + + initPictureRGB(); + } + + if (audio_st != null) { + // Find the decoder for the audio stream + AVCodec codec = avcodec_find_decoder_by_name(audioCodecName); + if (codec == null) { + codec = avcodec_find_decoder(audio_par.codec_id()); + } + if (codec == null) { + throw new Exception("avcodec_find_decoder() error: Unsupported audio format or codec not found: " + audio_par.codec_id() + "."); + } + + /* Allocate a codec context for the decoder */ + if ((audio_c = avcodec_alloc_context3(codec)) == null) { + throw new Exception("avcodec_alloc_context3() error: Could not allocate audio decoding context."); + } + + /* copy the stream parameters from the muxer */ + if ((ret = avcodec_parameters_to_context(audio_c, audio_st.codecpar())) < 0) { + releaseUnsafe(); + throw new Exception("avcodec_parameters_to_context() error " + ret + ": Could not copy the audio stream parameters."); + } + + options = new AVDictionary(null); + for (Entry e : audioOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + + // Enable multithreading when available + audio_c.thread_count(0); + + // Open audio codec + if ((ret = avcodec_open2(audio_c, codec, options)) < 0) { + throw new Exception("avcodec_open2() error " + ret + ": Could not open audio codec."); + } + av_dict_free(options); + + // Allocate audio samples frame + if ((samples_frame = av_frame_alloc()) == null) { + throw new Exception("av_frame_alloc() error: Could not allocate audio frame."); + } + + samples_ptr = new BytePointer[] { null }; + samples_buf = new Buffer[] { null }; + } + started = true; + + } + } + + private void initPictureRGB() { + int width = imageWidth > 0 ? imageWidth : video_c.width(); + int height = imageHeight > 0 ? imageHeight : video_c.height(); + + switch (imageMode) { + case COLOR: + case GRAY: + // If size changes I new allocation is needed -> free the old one. + if (image_ptr != null) { + // First kill all references, then free it. + image_buf = null; + BytePointer[] temp = image_ptr; + image_ptr = null; + av_free(temp[0]); + } + int fmt = getPixelFormat(); + + // work around bug in swscale: https://trac.ffmpeg.org/ticket/1031 + int align = 64; + int stride = width; + for (int i = 1; i <= align; i += i) { + stride = (width + (i - 1)) & ~(i - 1); + av_image_fill_linesizes(picture_rgb.linesize(), fmt, stride); + if ((picture_rgb.linesize(0) & (align - 1)) == 0) { + break; + } + } + + // Determine required buffer size and allocate buffer + int size = av_image_get_buffer_size(fmt, stride, height, 1); + image_ptr = new BytePointer[] { new BytePointer(av_malloc(size)).capacity(size) }; + image_buf = new Buffer[] { image_ptr[0].asBuffer() }; + + // Assign appropriate parts of buffer to image planes in picture_rgb + // Note that picture_rgb is an AVFrame, but AVFrame is a superset of AVPicture + av_image_fill_arrays(new PointerPointer(picture_rgb), picture_rgb.linesize(), image_ptr[0], fmt, stride, height, 1); + picture_rgb.format(fmt); + picture_rgb.width(width); + picture_rgb.height(height); + break; + + case RAW: + image_ptr = new BytePointer[] { null }; + image_buf = new Buffer[] { null }; + break; + + default: + assert false; + } + } + + @Override public void stop() throws Exception { + release(); + } + + @Override public synchronized void trigger() throws Exception { + if (oc == null || oc.isNull()) { + throw new Exception("Could not trigger: No AVFormatContext. (Has start() been called?)"); + } + if (pkt.stream_index() != -1) { + av_packet_unref(pkt); + pkt.stream_index(-1); + } + for (int i = 0; i < numBuffers+1; i++) { + if (av_read_frame(oc, pkt) < 0) { + return; + } + av_packet_unref(pkt); + } + } + + private void processImage() throws Exception { + frame.imageWidth = imageWidth > 0 ? imageWidth : video_c.width(); + frame.imageHeight = imageHeight > 0 ? imageHeight : video_c.height(); + frame.imageDepth = Frame.DEPTH_UBYTE; + switch (imageMode) { + case COLOR: + case GRAY: + // Deinterlace Picture + if (deinterlace) { + throw new Exception("Cannot deinterlace: Functionality moved to FFmpegFrameFilter."); + } + + // Has the size changed? + if (frame.imageWidth != picture_rgb.width() || frame.imageHeight != picture_rgb.height()) { + initPictureRGB(); + } + + // Copy "metadata" fields + av_frame_copy_props(picture_rgb, picture); + + // Convert the image into BGR or GRAY format that OpenCV uses + img_convert_ctx = sws_getCachedContext(img_convert_ctx, + video_c.width(), video_c.height(), video_c.pix_fmt(), + frame.imageWidth, frame.imageHeight, getPixelFormat(), + imageScalingFlags != 0 ? imageScalingFlags : SWS_BILINEAR, + null, null, (DoublePointer)null); + if (img_convert_ctx == null) { + throw new Exception("sws_getCachedContext() error: Cannot initialize the conversion context."); + } + + // Convert the image from its native format to RGB or GRAY + sws_scale(img_convert_ctx, new PointerPointer(picture), picture.linesize(), 0, + video_c.height(), new PointerPointer(picture_rgb), picture_rgb.linesize()); + frame.imageStride = picture_rgb.linesize(0); + frame.image = image_buf; + frame.opaque = picture_rgb; + break; + + case RAW: + frame.imageStride = picture.linesize(0); + BytePointer ptr = picture.data(0); + if (ptr != null && !ptr.equals(image_ptr[0])) { + image_ptr[0] = ptr.capacity(frame.imageHeight * frame.imageStride); + image_buf[0] = ptr.asBuffer(); + } + frame.image = image_buf; + frame.opaque = picture; + break; + + default: + assert false; + } + frame.image[0].limit(frame.imageHeight * frame.imageStride); + frame.imageChannels = frame.imageStride / frame.imageWidth; + } + + private void processSamples() throws Exception { + int ret; + + int sample_format = samples_frame.format(); + int planes = av_sample_fmt_is_planar(sample_format) != 0 ? (int)samples_frame.ch_layout().nb_channels() : 1; + int data_size = av_samples_get_buffer_size((IntPointer)null, audio_c.ch_layout().nb_channels(), + samples_frame.nb_samples(), audio_c.sample_fmt(), 1) / planes; + if (samples_buf == null || samples_buf.length != planes) { + samples_ptr = new BytePointer[planes]; + samples_buf = new Buffer[planes]; + } + frame.sampleRate = audio_c.sample_rate(); + frame.audioChannels = audio_c.ch_layout().nb_channels(); + frame.samples = samples_buf; + frame.opaque = samples_frame; + int sample_size = data_size / av_get_bytes_per_sample(sample_format); + for (int i = 0; i < planes; i++) { + BytePointer p = samples_frame.data(i); + if (!p.equals(samples_ptr[i]) || samples_ptr[i].capacity() < data_size) { + samples_ptr[i] = p.capacity(data_size); + ByteBuffer b = p.asBuffer(); + switch (sample_format) { + case AV_SAMPLE_FMT_U8: + case AV_SAMPLE_FMT_U8P: samples_buf[i] = b; break; + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: samples_buf[i] = b.asShortBuffer(); break; + case AV_SAMPLE_FMT_S32: + case AV_SAMPLE_FMT_S32P: samples_buf[i] = b.asIntBuffer(); break; + case AV_SAMPLE_FMT_FLT: + case AV_SAMPLE_FMT_FLTP: samples_buf[i] = b.asFloatBuffer(); break; + case AV_SAMPLE_FMT_DBL: + case AV_SAMPLE_FMT_DBLP: samples_buf[i] = b.asDoubleBuffer(); break; + default: assert false; + } + } + samples_buf[i].position(0).limit(sample_size); + } + + if (audio_c.ch_layout().nb_channels() != getAudioChannels() || audio_c.sample_fmt() != getSampleFormat() || audio_c.sample_rate() != getSampleRate()) { + if (samples_convert_ctx == null || samples_channels != getAudioChannels() || samples_format != getSampleFormat() || samples_rate != getSampleRate()) { + if (samples_convert_ctx == null) { + samples_convert_ctx = new SwrContext().retainReference(); + } + av_channel_layout_default(default_layout, getAudioChannels()); + if ((ret = swr_alloc_set_opts2(samples_convert_ctx, default_layout, getSampleFormat(), getSampleRate(), + audio_c.ch_layout(), audio_c.sample_fmt(), audio_c.sample_rate(), 0, null)) < 0) { + throw new Exception("swr_alloc_set_opts2() error " + ret + ": Cannot allocate the conversion context."); + } else if ((ret = swr_init(samples_convert_ctx)) < 0) { + throw new Exception("swr_init() error " + ret + ": Cannot initialize the conversion context."); + } + samples_channels = getAudioChannels(); + samples_format = getSampleFormat(); + samples_rate = getSampleRate(); + } + + int sample_size_in = samples_frame.nb_samples(); + int planes_out = av_sample_fmt_is_planar(samples_format) != 0 ? (int)samples_frame.ch_layout().nb_channels() : 1; + int sample_size_out = swr_get_out_samples(samples_convert_ctx, sample_size_in); + int sample_bytes_out = av_get_bytes_per_sample(samples_format); + int buffer_size_out = sample_size_out * sample_bytes_out * (planes_out > 1 ? 1 : samples_channels); + if (samples_buf_out == null || samples_buf.length != planes_out || samples_ptr_out[0].capacity() < buffer_size_out) { + for (int i = 0; samples_ptr_out != null && i < samples_ptr_out.length; i++) { + av_free(samples_ptr_out[i].position(0)); + } + samples_ptr_out = new BytePointer[planes_out]; + samples_buf_out = new Buffer[planes_out]; + + for (int i = 0; i < planes_out; i++) { + samples_ptr_out[i] = new BytePointer(av_malloc(buffer_size_out)).capacity(buffer_size_out); + ByteBuffer b = samples_ptr_out[i].asBuffer(); + switch (samples_format) { + case AV_SAMPLE_FMT_U8: + case AV_SAMPLE_FMT_U8P: samples_buf_out[i] = b; break; + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: samples_buf_out[i] = b.asShortBuffer(); break; + case AV_SAMPLE_FMT_S32: + case AV_SAMPLE_FMT_S32P: samples_buf_out[i] = b.asIntBuffer(); break; + case AV_SAMPLE_FMT_FLT: + case AV_SAMPLE_FMT_FLTP: samples_buf_out[i] = b.asFloatBuffer(); break; + case AV_SAMPLE_FMT_DBL: + case AV_SAMPLE_FMT_DBLP: samples_buf_out[i] = b.asDoubleBuffer(); break; + default: assert false; + } + } + } + frame.sampleRate = samples_rate; + frame.audioChannels = samples_channels; + frame.samples = samples_buf_out; + + if ((ret = swr_convert(samples_convert_ctx, plane_ptr.put(samples_ptr_out), sample_size_out, plane_ptr2.put(samples_ptr), sample_size_in)) < 0) { + throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples."); + } + for (int i = 0; i < planes_out; i++) { + samples_ptr_out[i].position(0).limit(ret * (planes_out > 1 ? 1 : samples_channels)); + samples_buf_out[i].position(0).limit(ret * (planes_out > 1 ? 1 : samples_channels)); + } + } + } + + public Frame grab() throws Exception { + return grabFrame(true, true, true, false, true); + } + public Frame grabImage() throws Exception { + return grabFrame(false, true, true, false, false); + } + public Frame grabSamples() throws Exception { + return grabFrame(true, false, true, false, false); + } + public Frame grabKeyFrame() throws Exception { + return grabFrame(false, true, true, true, false); + } + public Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, boolean keyFrames) throws Exception { + return grabFrame(doAudio, doVideo, doProcessing, keyFrames, true); + } + public synchronized Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, boolean keyFrames, boolean doData) throws Exception { + try (PointerScope scope = new PointerScope()) { + + if (oc == null || oc.isNull()) { + throw new Exception("Could not grab: No AVFormatContext. (Has start() been called?)"); + } else if ((!doVideo || video_st == null) && (!doAudio || audio_st == null) && !doData) { + return null; + } + if (!started) { + throw new Exception("start() was not called successfully!"); + } + + boolean videoFrameGrabbed = frameGrabbed && frame.image != null; + boolean audioFrameGrabbed = frameGrabbed && frame.samples != null; + boolean dataFrameGrabbed = frameGrabbed && frame.data != null; + frameGrabbed = false; + if (doVideo && videoFrameGrabbed) { + if (doProcessing) { + processImage(); + } + frame.keyFrame = picture.key_frame() != 0; + return frame; + } else if (doAudio && audioFrameGrabbed) { + if (doProcessing) { + processSamples(); + } + frame.keyFrame = samples_frame.key_frame() != 0; + return frame; + } else if (doData && dataFrameGrabbed) { + return frame; + } + + frame.keyFrame = false; + frame.imageWidth = 0; + frame.imageHeight = 0; + frame.imageDepth = 0; + frame.imageChannels = 0; + frame.imageStride = 0; + frame.image = null; + frame.sampleRate = 0; + frame.audioChannels = 0; + frame.samples = null; + frame.data = null; + frame.opaque = null; + frame.type = null; + + boolean done = false; + boolean readPacket = pkt.stream_index() == -1; + while (!done) { + int ret = 0; + if (readPacket) { + if (pkt.stream_index() != -1) { + // Free the packet that was allocated by av_read_frame + av_packet_unref(pkt); + pkt.stream_index(-1); + } + if ((ret = av_read_frame(oc, pkt)) < 0) { + if (ret == AVERROR_EAGAIN()) { + try { + Thread.sleep(10); + continue; + } catch (InterruptedException ex) { + // reset interrupt to be nice + Thread.currentThread().interrupt(); + return null; + } + } + if ((doVideo && video_st != null) || (doAudio && audio_st != null)) { + // The video or audio codec may have buffered some frames + pkt.stream_index(doVideo && video_st != null ? video_st.index() : audio_st.index()); + pkt.flags(AV_PKT_FLAG_KEY); + pkt.data(null); + pkt.size(0); + } else { + pkt.stream_index(-1); + return null; + } + } + } + + frame.streamIndex = pkt.stream_index(); + + // Is this a packet from the video stream? + if (doVideo && video_st != null && frame.streamIndex == video_st.index() + && (!keyFrames || pkt.flags() == AV_PKT_FLAG_KEY)) { + // Decode video frame + if (readPacket) { + ret = avcodec_send_packet(video_c, pkt); + if (pkt.data() == null && pkt.size() == 0) { + pkt.stream_index(-1); + } + if (ret == AVERROR_EAGAIN() || ret == AVERROR_EOF()) { + // The video codec may have buffered some frames + } else if (ret < 0) { + // Ignore errors to emulate the behavior of the old API + // throw new Exception("avcodec_send_packet() error " + ret + ": Error sending a video packet for decoding."); + } + } + + // Did we get a video frame? + while (!done) { + ret = avcodec_receive_frame(video_c, picture); + if (ret == AVERROR_EAGAIN() || ret == AVERROR_EOF()) { + if (pkt.data() == null && pkt.size() == 0) { + pkt.stream_index(-1); + doVideo = false; + if (doAudio) { + readPacket = false; + break; + } + return null; + } else { + readPacket = true; + break; + } + } else if (ret < 0) { + // Ignore errors to emulate the behavior of the old API + // throw new Exception("avcodec_receive_frame() error " + ret + ": Error during video decoding."); + readPacket = true; + break; + } + + if (!keyFrames || picture.pict_type() == AV_PICTURE_TYPE_I) { + long pts = picture.best_effort_timestamp(); + AVRational time_base = video_st.time_base(); + timestamp = 1000000L * pts * time_base.num() / time_base.den(); + long ts0 = oc.start_time() != AV_NOPTS_VALUE ? oc.start_time() : 0; + // best guess, AVCodecContext.frame_number = number of decoded frames... + frameNumber = (int)Math.round((timestamp - ts0) * getFrameRate() / 1000000L); + frame.image = image_buf; + if (doProcessing) { + processImage(); + } + /* the picture is allocated by the decoder. no need to + free it */ + done = true; + frame.timestamp = timestamp; + frame.keyFrame = picture.key_frame() != 0; + frame.pictType = (char)av_get_picture_type_char(picture.pict_type()); + frame.type = Frame.Type.VIDEO; + } + } + } else if (doAudio && audio_st != null && frame.streamIndex == audio_st.index()) { + // Decode audio frame + if (readPacket) { + ret = avcodec_send_packet(audio_c, pkt); + if (ret < 0) { + // Ignore errors to emulate the behavior of the old API + // throw new Exception("avcodec_send_packet() error " + ret + ": Error sending an audio packet for decoding."); + } + } + + // Did we get an audio frame? + while (!done) { + ret = avcodec_receive_frame(audio_c, samples_frame); + if (ret == AVERROR_EAGAIN() || ret == AVERROR_EOF()) { + if (pkt.data() == null && pkt.size() == 0) { + pkt.stream_index(-1); + doAudio = false; + return null; + } else { + readPacket = true; + break; + } + } else if (ret < 0) { + // Ignore errors to emulate the behavior of the old API + // throw new Exception("avcodec_receive_frame() error " + ret + ": Error during audio decoding."); + readPacket = true; + break; + } + + long pts = samples_frame.best_effort_timestamp(); + AVRational time_base = audio_st.time_base(); + timestamp = 1000000L * pts * time_base.num() / time_base.den(); + frame.samples = samples_buf; + /* if a frame has been decoded, output it */ + if (doProcessing) { + processSamples(); + } + done = true; + frame.timestamp = timestamp; + frame.keyFrame = samples_frame.key_frame() != 0; + frame.type = Frame.Type.AUDIO; + } + } else if (readPacket && doData + && frame.streamIndex > -1 && frame.streamIndex < streams.length + && streams[frame.streamIndex] != AVMEDIA_TYPE_VIDEO && streams[frame.streamIndex] != AVMEDIA_TYPE_AUDIO) { + // Export the stream byte data for non audio / video frames + frame.data = pkt.data().position(0).capacity(pkt.size()).asByteBuffer(); + frame.opaque = pkt; + done = true; + switch (streams[frame.streamIndex]) { + case AVMEDIA_TYPE_DATA: frame.type = Frame.Type.DATA; break; + case AVMEDIA_TYPE_SUBTITLE: frame.type = Frame.Type.SUBTITLE; break; + case AVMEDIA_TYPE_ATTACHMENT: frame.type = Frame.Type.ATTACHMENT; break; + default: frame.type = null; + } + } else { + // Current packet is not needed (different stream index required) + readPacket = true; + } + } + return frame; + + } + } + + public synchronized AVPacket grabPacket() throws Exception { + if (oc == null || oc.isNull()) { + throw new Exception("Could not grab: No AVFormatContext. (Has start() been called?)"); + } + if (!started) { + throw new Exception("start() was not called successfully!"); + } + + // Return the next frame of a stream. + if (av_read_frame(oc, pkt) < 0) { + return null; + } + + return pkt; + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java new file mode 100644 index 00000000..eb4224cf --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -0,0 +1,1442 @@ +/* + * Copyright (C) 2009-2024 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + * Based on the output-example.c file included in FFmpeg 0.6.5 + * as well as on the decoding_encoding.c file included in FFmpeg 0.11.1, + * and on the encode_video.c file included in FFmpeg 4.4, + * which are covered by the following copyright notice: + * + * Libavformat API example: Output a media file in any supported + * libavformat format. The default codecs are used. + * + * Copyright (c) 2001,2003 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.bytedeco.javacv; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.DoubleBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.FloatPointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerScope; +import org.bytedeco.javacpp.PointerPointer; +import org.bytedeco.javacpp.ShortPointer; + +import org.bytedeco.ffmpeg.avcodec.*; +import org.bytedeco.ffmpeg.avdevice.*; +import org.bytedeco.ffmpeg.avformat.*; +import org.bytedeco.ffmpeg.avutil.*; +import org.bytedeco.ffmpeg.swresample.*; +import org.bytedeco.ffmpeg.swscale.*; +import static org.bytedeco.ffmpeg.global.avcodec.*; +import static org.bytedeco.ffmpeg.global.avdevice.*; +import static org.bytedeco.ffmpeg.global.avformat.*; +import static org.bytedeco.ffmpeg.global.avutil.*; +import static org.bytedeco.ffmpeg.global.swresample.*; +import static org.bytedeco.ffmpeg.global.swscale.*; + +/** + * + * @author Samuel Audet + */ +public class FFmpegFrameRecorder extends FrameRecorder { + + public static class Exception extends FrameRecorder.Exception { + public Exception(String message) { super(message + " (For more details, make sure FFmpegLogCallback.set() has been called.)"); } + public Exception(String message, Throwable cause) { super(message, cause); } + } + + public static FFmpegFrameRecorder createDefault(File f, int w, int h) throws Exception { return new FFmpegFrameRecorder(f, w, h); } + public static FFmpegFrameRecorder createDefault(String f, int w, int h) throws Exception { return new FFmpegFrameRecorder(f, w, h); } + + private static Exception loadingException = null; + public static void tryLoad() throws Exception { + if (loadingException != null) { + throw loadingException; + } else { + try { + Loader.load(org.bytedeco.ffmpeg.global.avutil.class); + Loader.load(org.bytedeco.ffmpeg.global.swresample.class); + Loader.load(org.bytedeco.ffmpeg.global.avcodec.class); + Loader.load(org.bytedeco.ffmpeg.global.avformat.class); + Loader.load(org.bytedeco.ffmpeg.global.swscale.class); + + /* initialize libavcodec, and register all codecs and formats */ + av_jni_set_java_vm(Loader.getJavaVM(), null); + // avcodec_register_all(); + // av_register_all(); + avformat_network_init(); + + Loader.load(org.bytedeco.ffmpeg.global.avdevice.class); + avdevice_register_all(); + } catch (Throwable t) { + if (t instanceof Exception) { + throw loadingException = (Exception)t; + } else { + throw loadingException = new Exception("Failed to load " + FFmpegFrameRecorder.class, t); + } + } + } + } + + static { + try { + tryLoad(); + // FFmpegLockCallback.init(); + } catch (Exception ex) { } + } + + public FFmpegFrameRecorder(URL url, int audioChannels) { + this(url.toString(), 0, 0, audioChannels); + } + public FFmpegFrameRecorder(File file, int audioChannels) { + this(file, 0, 0, audioChannels); + } + public FFmpegFrameRecorder(String filename, int audioChannels) { + this(filename, 0, 0, audioChannels); + } + public FFmpegFrameRecorder(URL url, int imageWidth, int imageHeight) { + this(url.toString(), imageWidth, imageHeight, 0); + } + public FFmpegFrameRecorder(File file, int imageWidth, int imageHeight) { + this(file, imageWidth, imageHeight, 0); + } + public FFmpegFrameRecorder(String filename, int imageWidth, int imageHeight) { + this(filename, imageWidth, imageHeight, 0); + } + public FFmpegFrameRecorder(URL url, int imageWidth, int imageHeight, int audioChannels) { + this(url.toString(), imageWidth, imageHeight, audioChannels); + } + public FFmpegFrameRecorder(File file, int imageWidth, int imageHeight, int audioChannels) { + this(file.getAbsolutePath(), imageWidth, imageHeight, audioChannels); + } + public FFmpegFrameRecorder(String filename, int imageWidth, int imageHeight, int audioChannels) { + this.filename = filename; + this.imageWidth = imageWidth; + this.imageHeight = imageHeight; + this.audioChannels = audioChannels; + + this.pixelFormat = AV_PIX_FMT_NONE; + this.videoCodec = AV_CODEC_ID_NONE; + this.videoBitrate = 400000; + this.frameRate = 30; + + this.sampleFormat = AV_SAMPLE_FMT_NONE; + this.audioCodec = AV_CODEC_ID_NONE; + this.audioBitrate = 64000; + this.sampleRate = 44100; + + this.interleaved = true; + } + + public FFmpegFrameRecorder(OutputStream outputStream, int audioChannels) { + this(outputStream.toString(), audioChannels); + this.outputStream = outputStream; + this.closeOutputStream = true; + } + public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageHeight) { + this(outputStream.toString(), imageWidth, imageHeight); + this.outputStream = outputStream; + this.closeOutputStream = true; + } + public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageHeight, int audioChannels) { + this(outputStream.toString(), imageWidth, imageHeight, audioChannels); + this.outputStream = outputStream; + this.closeOutputStream = true; + } + public void release() throws Exception { + synchronized (org.bytedeco.ffmpeg.global.avcodec.class) { + releaseUnsafe(); + } + } + public synchronized void releaseUnsafe() throws Exception { + started = false; + + if (display_matrix != null) { + display_matrix.releaseReference(); + } + + if (plane_ptr != null && plane_ptr2 != null) { + plane_ptr.releaseReference(); + plane_ptr2.releaseReference(); + plane_ptr = plane_ptr2 = null; + } + + if (video_pkt != null && audio_pkt != null) { + video_pkt.releaseReference(); + audio_pkt.releaseReference(); + video_pkt = audio_pkt = null; + } + + if (default_layout != null) { + default_layout.releaseReference(); + default_layout = null; + } + + /* close each codec */ + if (video_c != null) { + avcodec_free_context(video_c); + video_c = null; + } + if (audio_c != null) { + avcodec_free_context(audio_c); + audio_c = null; + } + if (picture_buf != null) { + av_free(picture_buf); + picture_buf = null; + } + if (picture != null) { + av_frame_free(picture); + picture = null; + } + if (tmp_picture != null) { + av_frame_free(tmp_picture); + tmp_picture = null; + } + if (video_outbuf != null) { + av_free(video_outbuf); + video_outbuf = null; + } + if (frame != null) { + av_frame_free(frame); + frame = null; + } + if (samples_in != null) { + for (int i = 0; i < samples_in.length; i++) { + if (samples_in[i] != null) { + samples_in[i].releaseReference(); + } + } + samples_in = null; + } + if (samples_out != null) { + for (int i = 0; i < samples_out.length; i++) { + av_free(samples_out[i].position(0)); + } + samples_out = null; + } + if (audio_outbuf != null) { + av_free(audio_outbuf); + audio_outbuf = null; + } + if (video_st != null && video_st.metadata() != null) { + av_dict_free(video_st.metadata()); + video_st.metadata(null); + } + if (audio_st != null && audio_st.metadata() != null) { + av_dict_free(audio_st.metadata()); + audio_st.metadata(null); + } + video_st = null; + audio_st = null; + filename = null; + + AVFormatContext outputStreamKey = oc; + if (oc != null && !oc.isNull()) { + if (outputStream == null && (oformat.flags() & AVFMT_NOFILE) == 0) { + /* close the output file */ + avio_close(oc.pb()); + } + + /* free the streams */ + avformat_free_context(oc); + oc = null; + } + + if (img_convert_ctx != null) { + sws_freeContext(img_convert_ctx); + img_convert_ctx = null; + } + + if (samples_convert_ctx != null) { + swr_free(samples_convert_ctx); + samples_convert_ctx.releaseReference(); + samples_convert_ctx = null; + } + + if (outputStream != null) { + try { + if (closeOutputStream) { + outputStream.close(); + } + } catch (IOException ex) { + throw new Exception("Error on OutputStream.close(): ", ex); + } finally { + outputStream = null; + outputStreams.remove(outputStreamKey); + if (avio != null) { + if (avio.buffer() != null) { + av_free(avio.buffer()); + avio.buffer(null); + } + av_free(avio); + avio = null; + } + } + } + } + @Override protected void finalize() throws Throwable { + super.finalize(); + release(); + } + + static Map outputStreams = Collections.synchronizedMap(new HashMap()); + + static class WriteCallback extends Write_packet_Pointer_BytePointer_int { + @Override public int call(Pointer opaque, BytePointer buf, int buf_size) { + try { + byte[] b = new byte[buf_size]; + OutputStream os = outputStreams.get(opaque); + buf.get(b, 0, buf_size); + os.write(b, 0, buf_size); + return buf_size; + } + catch (Throwable t) { + System.err.println("Error on OutputStream.write(): " + t); + return -1; + } + } + } + + static WriteCallback writeCallback = new WriteCallback().retainReference(); + + static class SeekCallback extends Seek_Pointer_long_int { + + @Override public long call(Pointer opaque, long offset, int whence) { + try { + OutputStream os = outputStreams.get(opaque); + ((Seekable)os).seek(offset, whence); + return 0; + } + catch (Throwable t) { + System.err.println("Error on OutputStream.seek(): " + t); + return -1; + } + } + } + + static SeekCallback seekCallback = new SeekCallback().retainReference(); + + private OutputStream outputStream; + private boolean closeOutputStream; + private AVIOContext avio; + private String filename; + private AVFrame picture, tmp_picture; + private BytePointer picture_buf; + private BytePointer video_outbuf; + private int video_outbuf_size; + private AVFrame frame; + private Pointer[] samples_in; + private BytePointer[] samples_out; + private BytePointer audio_outbuf; + private int audio_outbuf_size; + private int audio_input_frame_size; + private AVOutputFormat oformat; + private AVFormatContext oc; + private AVCodec video_codec, audio_codec; + private AVCodecContext video_c, audio_c; + private AVStream video_st, audio_st; + private SwsContext img_convert_ctx; + private SwrContext samples_convert_ctx; + private int samples_channels, samples_format, samples_rate; + private PointerPointer plane_ptr, plane_ptr2; + private AVPacket video_pkt, audio_pkt; + private int[] got_video_packet, got_audio_packet; + private AVFormatContext ifmt_ctx; + private IntPointer display_matrix; + private AVChannelLayout default_layout; + + private volatile boolean started = false; + + public boolean isCloseOutputStream() { + return closeOutputStream; + } + public void setCloseOutputStream(boolean closeOutputStream) { + this.closeOutputStream = closeOutputStream; + } + + /** Sets the rotation in degrees to the side data of the video stream. */ + public void setDisplayRotation(double angle) { + if (display_matrix == null) { + display_matrix = new IntPointer(9).retainReference(); + } + av_display_rotation_set(display_matrix, -angle); + setVideoSideData("Display Matrix", display_matrix.asByteBuffer()); + } + + @Override public int getFrameNumber() { + return picture == null ? super.getFrameNumber() : (int)picture.pts(); + } + @Override public void setFrameNumber(int frameNumber) { + if (picture == null) { super.setFrameNumber(frameNumber); } else { picture.pts(frameNumber); } + } + + /** Returns best guess for timestamp in microseconds... */ + @Override public long getTimestamp() { + return Math.round(getFrameNumber() * 1000000L / getFrameRate()); + } + @Override public void setTimestamp(long timestamp) { + setFrameNumber((int)Math.round(timestamp * getFrameRate() / 1000000L)); + } + + public void start(AVFormatContext inputFormatContext) throws Exception { + this.ifmt_ctx = inputFormatContext; + start(); + } + + @Override public void start() throws Exception { + synchronized (org.bytedeco.ffmpeg.global.avcodec.class) { + startUnsafe(); + } + } + + public synchronized void startUnsafe() throws Exception { + try (PointerScope scope = new PointerScope()) { + + if (oc != null && !oc.isNull()) { + throw new Exception("start() has already been called: Call stop() before calling start() again."); + } + + int ret; + picture = null; + tmp_picture = null; + picture_buf = null; + frame = null; + video_outbuf = null; + audio_outbuf = null; + oc = new AVFormatContext(null); + video_c = null; + audio_c = null; + video_st = null; + audio_st = null; + plane_ptr = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS).retainReference(); + plane_ptr2 = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS).retainReference(); + video_pkt = new AVPacket().retainReference(); + audio_pkt = new AVPacket().retainReference(); + got_video_packet = new int[1]; + got_audio_packet = new int[1]; + default_layout = new AVChannelLayout().retainReference(); + + /* auto detect the output format from the name. */ + String format_name = format == null || format.length() == 0 ? null : format; + if ((oformat = av_guess_format(format_name, filename, null)) == null) { + int proto = filename.indexOf("://"); + if (proto > 0) { + format_name = filename.substring(0, proto); + } + if ((oformat = av_guess_format(format_name, filename, null)) == null) { + throw new Exception("av_guess_format() error: Could not guess output format for \"" + filename + "\" and " + format + " format."); + } + } + format_name = oformat.name().getString(); + + /* allocate the output media context */ + if (avformat_alloc_output_context2(oc, null, format_name, filename) < 0) { + throw new Exception("avformat_alloc_context2() error:\tCould not allocate format context"); + } + + if (outputStream != null) { + avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 1, oc, null, writeCallback, outputStream instanceof Seekable ? seekCallback : null); + oc.pb(avio); + + filename = outputStream.toString(); + outputStreams.put(oc, outputStream); + } + oc.oformat(oformat); + oc.url(new BytePointer(av_malloc(filename.getBytes().length + 1)).putString(filename)); + oc.max_delay(maxDelay); + + /* add the audio and video streams using the format codecs + and initialize the codecs */ + AVStream inpVideoStream = null, inpAudioStream = null; + if (ifmt_ctx != null) { + // get input video and audio stream indices from ifmt_ctx + for (int idx = 0; idx < ifmt_ctx.nb_streams(); idx++) { + AVStream inputStream = ifmt_ctx.streams(idx); + if (inputStream.codecpar().codec_type() == AVMEDIA_TYPE_VIDEO) { + inpVideoStream = inputStream; + videoCodec = inpVideoStream.codecpar().codec_id(); + if (inpVideoStream.r_frame_rate().num() != AV_NOPTS_VALUE && inpVideoStream.r_frame_rate().den() != 0) { + frameRate = (inpVideoStream.r_frame_rate().num())*1.0d / (inpVideoStream.r_frame_rate().den()); + } + + } else if (inputStream.codecpar().codec_type() == AVMEDIA_TYPE_AUDIO) { + inpAudioStream = inputStream; + audioCodec = inpAudioStream.codecpar().codec_id(); + } + } + } + + if (imageWidth > 0 && imageHeight > 0) { + if (videoCodec == AV_CODEC_ID_NONE) { + videoCodec = oformat.video_codec(); + } + // if (videoCodec != AV_CODEC_ID_NONE) { + // oformat.video_codec(videoCodec); + // } else if ("flv".equals(format_name)) { + // oformat.video_codec(AV_CODEC_ID_FLV1); + // } else if ("mp4".equals(format_name)) { + // oformat.video_codec(AV_CODEC_ID_MPEG4); + // } else if ("3gp".equals(format_name)) { + // oformat.video_codec(AV_CODEC_ID_H263); + // } else if ("avi".equals(format_name)) { + // oformat.video_codec(AV_CODEC_ID_HUFFYUV); + // } + + /* find the video encoder */ + if ((video_codec = avcodec_find_encoder_by_name(videoCodecName)) == null && + (video_codec = avcodec_find_encoder(videoCodec)) == null) { + releaseUnsafe(); + throw new Exception("avcodec_find_encoder() error: Video codec not found."); + } + // oformat.video_codec(video_codec.id()); + + AVRational frame_rate = av_d2q(frameRate, 1001000); + AVRational supported_framerates = video_codec.supported_framerates(); + if (supported_framerates != null) { + int idx = av_find_nearest_q_idx(frame_rate, supported_framerates); + frame_rate = supported_framerates.position(idx); + } + + /* add a video output stream */ + if ((video_st = avformat_new_stream(oc, null)) == null) { + releaseUnsafe(); + throw new Exception("avformat_new_stream() error: Could not allocate video stream."); + } + + if ((video_c = avcodec_alloc_context3(video_codec)) == null) { + releaseUnsafe(); + throw new Exception("avcodec_alloc_context3() error: Could not allocate video encoding context."); + } + + if (inpVideoStream != null) { + if ((ret = avcodec_parameters_copy(video_st.codecpar(), inpVideoStream.codecpar())) < 0) { + releaseUnsafe(); + throw new Exception("avcodec_parameters_copy() error " + ret + ": Failed to copy video stream codec parameters from input to output"); + } + + videoBitrate = (int)inpVideoStream.codecpar().bit_rate(); + pixelFormat = inpVideoStream.codecpar().format(); + aspectRatio = inpVideoStream.codecpar().sample_aspect_ratio().num()*1.0d/ inpVideoStream.codecpar().sample_aspect_ratio().den(); + // videoQuality = inpVideoStream.codecpar().global_quality(); + video_c.codec_tag(0); + } + + video_c.codec_id(video_codec.id()); + video_c.codec_type(AVMEDIA_TYPE_VIDEO); + + + /* put sample parameters */ + video_c.bit_rate(videoBitrate); + /* resolution must be a multiple of two. Scale height to maintain the aspect ratio. */ + if (imageWidth % 2 == 1) { + int roundedWidth = imageWidth + 1; + imageHeight = (roundedWidth * imageHeight + imageWidth / 2) / imageWidth; + imageWidth = roundedWidth; + } + video_c.width(imageWidth); + video_c.height(imageHeight); + if (aspectRatio > 0) { + AVRational r = av_d2q(aspectRatio, 255); + video_c.sample_aspect_ratio(r); + video_st.sample_aspect_ratio(r); + } + /* time base: this is the fundamental unit of time (in seconds) in terms + of which frame timestamps are represented. for fixed-fps content, + timebase should be 1/framerate and timestamp increments should be + identically 1. */ + AVRational time_base = av_inv_q(frame_rate); + video_c.time_base(time_base); + video_st.time_base(time_base); + video_st.avg_frame_rate(frame_rate); + // video_st.codec().time_base(time_base); // "deprecated", but this is actually required + if (gopSize >= 0) { + video_c.gop_size(gopSize); /* emit one intra frame every gopSize frames at most */ + } + if (videoQuality >= 0) { + video_c.flags(video_c.flags() | AV_CODEC_FLAG_QSCALE); + video_c.global_quality((int)Math.round(FF_QP2LAMBDA * videoQuality)); + } + + if (pixelFormat != AV_PIX_FMT_NONE) { + video_c.pix_fmt(pixelFormat); + } else if (video_c.codec_id() == AV_CODEC_ID_RAWVIDEO || video_c.codec_id() == AV_CODEC_ID_PNG || + video_c.codec_id() == AV_CODEC_ID_HUFFYUV || video_c.codec_id() == AV_CODEC_ID_FFV1) { + video_c.pix_fmt(AV_PIX_FMT_RGB32); // appropriate for common lossless formats + } else if (video_c.codec_id() == AV_CODEC_ID_JPEGLS) { + video_c.pix_fmt(AV_PIX_FMT_BGR24); + } else if (video_c.codec_id() == AV_CODEC_ID_MJPEG || video_c.codec_id() == AV_CODEC_ID_MJPEGB) { + video_c.pix_fmt(AV_PIX_FMT_YUVJ420P); + } else { + video_c.pix_fmt(AV_PIX_FMT_YUV420P); // lossy, but works with about everything + } + + if (video_c.codec_id() == AV_CODEC_ID_MPEG2VIDEO) { + /* just for testing, we also add B frames */ + video_c.max_b_frames(2); + } else if (video_c.codec_id() == AV_CODEC_ID_MPEG1VIDEO) { + /* Needed to avoid using macroblocks in which some coeffs overflow. + This does not happen with normal video, it just happens here as + the motion of the chroma plane does not match the luma plane. */ + video_c.mb_decision(2); + } else if (video_c.codec_id() == AV_CODEC_ID_H263) { + // H.263 does not support any other resolution than the following + if (imageWidth <= 128 && imageHeight <= 96) { + video_c.width(128).height(96); + } else if (imageWidth <= 176 && imageHeight <= 144) { + video_c.width(176).height(144); + } else if (imageWidth <= 352 && imageHeight <= 288) { + video_c.width(352).height(288); + } else if (imageWidth <= 704 && imageHeight <= 576) { + video_c.width(704).height(576); + } else { + video_c.width(1408).height(1152); + } + } else if (video_c.codec_id() == AV_CODEC_ID_H264) { + // default to constrained baseline to produce content that plays back on anything, + // without any significant tradeoffs for most use cases + video_c.profile(AVCodecContext.FF_PROFILE_H264_CONSTRAINED_BASELINE); + } + + // some formats want stream headers to be separate + if ((oformat.flags() & AVFMT_GLOBALHEADER) != 0) { + video_c.flags(video_c.flags() | AV_CODEC_FLAG_GLOBAL_HEADER); + } + + if ((video_codec.capabilities() & AV_CODEC_CAP_EXPERIMENTAL) != 0) { + video_c.strict_std_compliance(FF_COMPLIANCE_EXPERIMENTAL); + } + + if (maxBFrames >= 0) { + video_c.max_b_frames(maxBFrames); + video_c.has_b_frames(maxBFrames == 0 ? 0 : 1); + } + + if (trellis >= 0) { + video_c.trellis(trellis); + } + } + + /* + * add an audio output stream + */ + if (audioChannels > 0 && audioBitrate > 0 && sampleRate > 0) { + if (audioCodec == AV_CODEC_ID_NONE) { + audioCodec = oformat.audio_codec(); + } + // if (audioCodec != AV_CODEC_ID_NONE) { + // oformat.audio_codec(audioCodec); + // } else if ("flv".equals(format_name) || "mp4".equals(format_name) || "3gp".equals(format_name)) { + // oformat.audio_codec(AV_CODEC_ID_AAC); + // } else if ("avi".equals(format_name)) { + // oformat.audio_codec(AV_CODEC_ID_PCM_S16LE); + // } + + /* find the audio encoder */ + if ((audio_codec = avcodec_find_encoder_by_name(audioCodecName)) == null && + (audio_codec = avcodec_find_encoder(audioCodec)) == null) { + releaseUnsafe(); + throw new Exception("avcodec_find_encoder() error: Audio codec not found."); + } + // oformat.audio_codec(audio_codec.id()); + + AVRational sample_rate = av_d2q(sampleRate, 1001000); + + if ((audio_st = avformat_new_stream(oc, null)) == null) { + releaseUnsafe(); + throw new Exception("avformat_new_stream() error: Could not allocate audio stream."); + } + + if ((audio_c = avcodec_alloc_context3(audio_codec)) == null) { + releaseUnsafe(); + throw new Exception("avcodec_alloc_context3() error: Could not allocate audio encoding context."); + } + + if (inpAudioStream != null && audioChannels > 0) { + if ((ret = avcodec_parameters_copy(audio_st.codecpar(), inpAudioStream.codecpar())) < 0) { + throw new Exception("avcodec_parameters_copy() error " + ret + ": Failed to copy audio stream codec parameters from input to output"); + } + + audioBitrate = (int) inpAudioStream.codecpar().bit_rate(); + sampleRate = inpAudioStream.codecpar().sample_rate(); + audioChannels = inpAudioStream.codecpar().ch_layout().nb_channels(); + sampleFormat = inpAudioStream.codecpar().format(); + // audioQuality = inpAudioStream.codecpar().global_quality(); + audio_c.codec_tag(0); + // audio_st.pts(inpAudioStream.pts()); + audio_st.duration(inpAudioStream.duration()); + audio_st.time_base().num(inpAudioStream.time_base().num()); + audio_st.time_base().den(inpAudioStream.time_base().den()); + } + + audio_c.codec_id(audio_codec.id()); + audio_c.codec_type(AVMEDIA_TYPE_AUDIO); + + + /* put sample parameters */ + audio_c.bit_rate(audioBitrate); + audio_c.sample_rate(sampleRate); + av_channel_layout_default(default_layout, audioChannels); + audio_c.ch_layout(default_layout); + if (sampleFormat != AV_SAMPLE_FMT_NONE) { + audio_c.sample_fmt(sampleFormat); + } else { + // use AV_SAMPLE_FMT_S16 by default, if available + audio_c.sample_fmt(AV_SAMPLE_FMT_FLTP); + IntPointer formats = audio_c.codec().sample_fmts(); + for (int i = 0; formats.get(i) != -1; i++) { + if (formats.get(i) == AV_SAMPLE_FMT_S16) { + audio_c.sample_fmt(AV_SAMPLE_FMT_S16); + break; + } + } + } + AVRational time_base = av_inv_q(sample_rate); + audio_c.time_base(time_base); + audio_st.time_base(time_base); + // audio_st.codec().time_base(time_base); // "deprecated", but this is actually required + switch (audio_c.sample_fmt()) { + case AV_SAMPLE_FMT_U8: + case AV_SAMPLE_FMT_U8P: audio_c.bits_per_raw_sample(8); break; + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: audio_c.bits_per_raw_sample(16); break; + case AV_SAMPLE_FMT_S32: + case AV_SAMPLE_FMT_S32P: audio_c.bits_per_raw_sample(32); break; + case AV_SAMPLE_FMT_FLT: + case AV_SAMPLE_FMT_FLTP: audio_c.bits_per_raw_sample(32); break; + case AV_SAMPLE_FMT_DBL: + case AV_SAMPLE_FMT_DBLP: audio_c.bits_per_raw_sample(64); break; + default: assert false; + } + if (audioQuality >= 0) { + audio_c.flags(audio_c.flags() | AV_CODEC_FLAG_QSCALE); + audio_c.global_quality((int)Math.round(FF_QP2LAMBDA * audioQuality)); + } + + // some formats want stream headers to be separate + if ((oformat.flags() & AVFMT_GLOBALHEADER) != 0) { + audio_c.flags(audio_c.flags() | AV_CODEC_FLAG_GLOBAL_HEADER); + } + + if ((audio_codec.capabilities() & AV_CODEC_CAP_EXPERIMENTAL) != 0) { + audio_c.strict_std_compliance(FF_COMPLIANCE_EXPERIMENTAL); + } + } + + /* now that all the parameters are set, we can open the audio and + video codecs and allocate the necessary encode buffers */ + if (video_st != null && inpVideoStream == null) { + AVDictionary options = new AVDictionary(null); + if (videoQuality >= 0) { + av_dict_set(options, "crf", "" + videoQuality, 0); + } + for (Entry e : videoOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + + // Enable multithreading when available + video_c.thread_count(0); + + /* open the codec */ + if ((ret = avcodec_open2(video_c, video_codec, options)) < 0) { + releaseUnsafe(); + av_dict_free(options); + throw new Exception("avcodec_open2() error " + ret + ": Could not open video codec."); + } + av_dict_free(options); + + video_outbuf = null; + // if ((oformat.flags() & AVFMT_RAWPICTURE) == 0) { + // /* allocate output buffer */ + // /* XXX: API change will be done */ + // /* buffers passed into lav* can be allocated any way you prefer, + // as long as they're aligned enough for the architecture, and + // they're freed appropriately (such as using av_free for buffers + // allocated with av_malloc) */ + // video_outbuf_size = Math.max(256 * 1024, 8 * video_c.width() * video_c.height()); // a la ffmpeg.c + // video_outbuf = new BytePointer(av_malloc(video_outbuf_size)); + // } + + /* allocate the encoded raw picture */ + if ((picture = av_frame_alloc()) == null) { + releaseUnsafe(); + throw new Exception("av_frame_alloc() error: Could not allocate picture."); + } + picture.pts(0); // magic required by libx264 + + int size = av_image_get_buffer_size(video_c.pix_fmt(), video_c.width(), video_c.height(), 1); + if ((picture_buf = new BytePointer(av_malloc(size))).isNull()) { + releaseUnsafe(); + throw new Exception("av_malloc() error: Could not allocate picture buffer."); + } + + /* if the output format is not equal to the image format, then a temporary + picture is needed too. It is then converted to the required output format */ + if ((tmp_picture = av_frame_alloc()) == null) { + releaseUnsafe(); + throw new Exception("av_frame_alloc() error: Could not allocate temporary picture."); + } + + /* copy the stream parameters to the muxer */ + if ((ret = avcodec_parameters_from_context(video_st.codecpar(), video_c)) < 0) { + releaseUnsafe(); + throw new Exception("avcodec_parameters_from_context() error " + ret + ": Could not copy the video stream parameters."); + } + + AVDictionary metadata = new AVDictionary(null); + for (Entry e : videoMetadata.entrySet()) { + av_dict_set(metadata, new BytePointer(e.getKey(), charset), new BytePointer(e.getValue(), charset), 0); + } + video_st.metadata(metadata); + + for (Entry e : videoSideData.entrySet()) { + int type = -1; + for (int i = 0; i < AV_PKT_DATA_NB; i++) { + BytePointer s = av_packet_side_data_name(i); + if (s != null && !s.isNull() && e.getKey().equals(s.getString())) { + type = i; + break; + } + } + Pointer p = new Pointer(e.getValue()); + BytePointer b = av_stream_new_side_data(video_st, type, p.capacity()); + if (b != null && !b.isNull()) { + b.capacity(p.capacity()).put(p); + } + } + } + + if (audio_st != null && inpAudioStream == null) { + AVDictionary options = new AVDictionary(null); + if (audioQuality >= 0) { + av_dict_set(options, "crf", "" + audioQuality, 0); + } + for (Entry e : audioOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + + // Enable multithreading when available + audio_c.thread_count(0); + + /* open the codec */ + if ((ret = avcodec_open2(audio_c, audio_codec, options)) < 0) { + releaseUnsafe(); + av_dict_free(options); + throw new Exception("avcodec_open2() error " + ret + ": Could not open audio codec."); + } + av_dict_free(options); + + audio_outbuf_size = 256 * 1024; + audio_outbuf = new BytePointer(av_malloc(audio_outbuf_size)); + + /* ugly hack for PCM codecs (will be removed ASAP with new PCM + support to compute the input frame size in samples */ + if (audio_c.frame_size() <= 1) { + audio_outbuf_size = AV_INPUT_BUFFER_MIN_SIZE; + audio_input_frame_size = audio_outbuf_size / audio_c.ch_layout().nb_channels(); + switch (audio_c.codec_id()) { + case AV_CODEC_ID_PCM_S16LE: + case AV_CODEC_ID_PCM_S16BE: + case AV_CODEC_ID_PCM_U16LE: + case AV_CODEC_ID_PCM_U16BE: + audio_input_frame_size >>= 1; + break; + default: + break; + } + } else { + audio_input_frame_size = audio_c.frame_size(); + } + //int bufferSize = audio_input_frame_size * audio_c.bits_per_raw_sample()/8 * audio_c.ch_layout().nb_channels(); + int planes = av_sample_fmt_is_planar(audio_c.sample_fmt()) != 0 ? (int)audio_c.ch_layout().nb_channels() : 1; + int data_size = av_samples_get_buffer_size((IntPointer)null, audio_c.ch_layout().nb_channels(), + audio_input_frame_size, audio_c.sample_fmt(), 1) / planes; + samples_out = new BytePointer[planes]; + for (int i = 0; i < samples_out.length; i++) { + samples_out[i] = new BytePointer(av_malloc(data_size)).capacity(data_size); + } + samples_in = new Pointer[AVFrame.AV_NUM_DATA_POINTERS]; + + /* allocate the audio frame */ + if ((frame = av_frame_alloc()) == null) { + releaseUnsafe(); + throw new Exception("av_frame_alloc() error: Could not allocate audio frame."); + } + frame.pts(0); // magic required by libvorbis and webm + + /* copy the stream parameters to the muxer */ + if ((ret = avcodec_parameters_from_context(audio_st.codecpar(), audio_c)) < 0) { + releaseUnsafe(); + throw new Exception("avcodec_parameters_from_context() error " + ret + ": Could not copy the audio stream parameters."); + } + + AVDictionary metadata = new AVDictionary(null); + for (Entry e : audioMetadata.entrySet()) { + av_dict_set(metadata, new BytePointer(e.getKey(), charset), new BytePointer(e.getValue(), charset), 0); + } + audio_st.metadata(metadata); + + for (Entry e : audioSideData.entrySet()) { + int type = -1; + for (int i = 0; i < AV_PKT_DATA_NB; i++) { + BytePointer s = av_packet_side_data_name(i); + if (s != null && !s.isNull() && e.getKey().equals(s.getString())) { + type = i; + break; + } + } + Pointer p = new Pointer(e.getValue()); + BytePointer b = av_stream_new_side_data(audio_st, type, p.capacity()); + if (b != null && !b.isNull()) { + b.capacity(p.capacity()).put(p); + } + } + } + + AVDictionary options = new AVDictionary(null); + for (Entry e : this.options.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + + /* open the output file, if needed */ + if (outputStream == null && (oformat.flags() & AVFMT_NOFILE) == 0) { + AVIOContext pb = new AVIOContext(null); + if ((ret = avio_open2(pb, filename, AVIO_FLAG_WRITE, null, options)) < 0) { + String errorMsg = "avio_open2 error() error " + ret + ": Could not open '" + filename + "'"; + releaseUnsafe(); + av_dict_free(options); + throw new Exception(errorMsg); + } + oc.pb(pb); + } + + AVDictionary metadata = new AVDictionary(null); + for (Entry e : this.metadata.entrySet()) { + av_dict_set(metadata, new BytePointer(e.getKey(), charset), new BytePointer(e.getValue(), charset), 0); + } + /* write the stream header, if any */ + if ((ret = avformat_write_header(oc.metadata(metadata), options)) < 0) { + String errorMsg = "avformat_write_header error() error " + ret + ": Could not write header to '" + filename + "'"; + releaseUnsafe(); + av_dict_free(options); + throw new Exception(errorMsg); + } + av_dict_free(options); + + if (av_log_get_level() >= AV_LOG_INFO) { + av_dump_format(oc, 0, filename, 1); + } + + started = true; + + } + } + + public synchronized void flush() throws Exception { + synchronized (oc) { + /* flush all the buffers */ + while (video_st != null && ifmt_ctx == null && recordImage(0, 0, 0, 0, 0, AV_PIX_FMT_NONE, (Buffer[])null)); + while (audio_st != null && ifmt_ctx == null && recordSamples(0, 0, (Buffer[])null)); + + if (interleaved && (video_st != null || audio_st != null)) { + av_interleaved_write_frame(oc, null); + } else { + av_write_frame(oc, null); + } + } + } + + public void stop() throws Exception { + if (oc != null) { + try { + flush(); + + /* write the trailer, if any */ + av_write_trailer(oc); + } finally { + release(); + } + } + } + + @Override public void record(Frame frame) throws Exception { + record(frame, frame != null && frame.opaque instanceof AVFrame ? ((AVFrame)frame.opaque).format() : AV_PIX_FMT_NONE); + } + public synchronized void record(Frame frame, int pixelFormat) throws Exception { + if (frame == null || (frame.image == null && frame.samples == null && frame.data == null)) { + recordImage(0, 0, 0, 0, 0, pixelFormat, (Buffer[])null); + } else { + if (frame.image != null) { + frame.keyFrame = recordImage(frame.imageWidth, frame.imageHeight, frame.imageDepth, + frame.imageChannels, frame.imageStride, pixelFormat, frame.image); + } + if (frame.samples != null) { + frame.keyFrame = recordSamples(frame.sampleRate, frame.audioChannels, frame.samples); + } + } + } + + public synchronized boolean recordImage(int width, int height, int depth, int channels, int stride, int pixelFormat, Buffer ... image) throws Exception { + try (PointerScope scope = new PointerScope()) { + + if (video_st == null) { + throw new Exception("No video output stream (Is imageWidth > 0 && imageHeight > 0 and has start() been called?)"); + } + if (!started) { + throw new Exception("start() was not called successfully!"); + } + int ret; + + if (image == null || image.length == 0) { + /* no more frame to compress. The codec has a latency of a few + frames if using B frames, so we get the last frames by + passing the same picture again */ + } else { + int step = stride * Math.abs(depth) / 8; + BytePointer data = image[0] instanceof ByteBuffer + ? new BytePointer((ByteBuffer)image[0]).position(0) + : new BytePointer(new Pointer(image[0]).position(0)); + + if (pixelFormat == AV_PIX_FMT_NONE) { + if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 3) { + pixelFormat = AV_PIX_FMT_BGR24; + } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 1) { + pixelFormat = AV_PIX_FMT_GRAY8; + } else if ((depth == Frame.DEPTH_USHORT || depth == Frame.DEPTH_SHORT) && channels == 1) { + pixelFormat = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN) ? + AV_PIX_FMT_GRAY16BE : AV_PIX_FMT_GRAY16LE; + } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 4) { + pixelFormat = AV_PIX_FMT_RGBA; + } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 2) { + pixelFormat = AV_PIX_FMT_NV21; // Android's camera capture format + } else { + throw new Exception("Could not guess pixel format of image: depth=" + depth + ", channels=" + channels); + } + } + + if (pixelFormat == AV_PIX_FMT_NV21) { + step = width; + } + + if (video_c.pix_fmt() != pixelFormat || video_c.width() != width || video_c.height() != height) { + /* convert to the codec pixel format if needed */ + img_convert_ctx = sws_getCachedContext(img_convert_ctx, width, height, pixelFormat, + video_c.width(), video_c.height(), video_c.pix_fmt(), + imageScalingFlags != 0 ? imageScalingFlags : SWS_BILINEAR, + null, null, (DoublePointer)null); + if (img_convert_ctx == null) { + throw new Exception("sws_getCachedContext() error: Cannot initialize the conversion context."); + } + av_image_fill_arrays(new PointerPointer(tmp_picture), tmp_picture.linesize(), data, pixelFormat, width, height, 1); + av_image_fill_arrays(new PointerPointer(picture), picture.linesize(), picture_buf, video_c.pix_fmt(), video_c.width(), video_c.height(), 1); + tmp_picture.linesize(0, step); + tmp_picture.format(pixelFormat); + tmp_picture.width(width); + tmp_picture.height(height); + picture.format(video_c.pix_fmt()); + picture.width(video_c.width()); + picture.height(video_c.height()); + sws_scale(img_convert_ctx, new PointerPointer(tmp_picture), tmp_picture.linesize(), + 0, height, new PointerPointer(picture), picture.linesize()); + } else { + av_image_fill_arrays(new PointerPointer(picture), picture.linesize(), data, pixelFormat, width, height, 1); + picture.linesize(0, step); + picture.format(pixelFormat); + picture.width(width); + picture.height(height); + } + } + + // if ((oformat.flags() & AVFMT_RAWPICTURE) != 0) { + // if (image == null || image.length == 0) { + // return false; + // } + // /* raw video case. The API may change slightly in the future for that? */ + // av_init_packet(video_pkt); + // video_pkt.flags(video_pkt.flags() | AV_PKT_FLAG_KEY); + // video_pkt.stream_index(video_st.index()); + // video_pkt.data(new BytePointer(picture)); + // video_pkt.size(Loader.sizeof(AVFrame.class)); + // } else { + /* encode the image */ + picture.quality(video_c.global_quality()); + if ((ret = avcodec_send_frame(video_c, image == null || image.length == 0 ? null : picture)) < 0 + && image != null && image.length != 0) { + throw new Exception("avcodec_send_frame() error " + ret + ": Error sending a video frame for encoding."); + } + picture.pts(picture.pts() + 1); // magic required by libx264 + + /* if zero size, it means the image was buffered */ + got_video_packet[0] = 0; + while (ret >= 0) { + av_new_packet(video_pkt, video_outbuf_size); + ret = avcodec_receive_packet(video_c, video_pkt); + if (ret == AVERROR_EAGAIN() || ret == AVERROR_EOF()) { + av_packet_unref(video_pkt); + break; + } else if (ret < 0) { + av_packet_unref(video_pkt); + throw new Exception("avcodec_receive_packet() error " + ret + ": Error during video encoding."); + } + got_video_packet[0] = 1; + + if (video_pkt.pts() != AV_NOPTS_VALUE) { + video_pkt.pts(av_rescale_q(video_pkt.pts(), video_c.time_base(), video_st.time_base())); + } + if (video_pkt.dts() != AV_NOPTS_VALUE) { + video_pkt.dts(av_rescale_q(video_pkt.dts(), video_c.time_base(), video_st.time_base())); + } + video_pkt.stream_index(video_st.index()); + + /* write the compressed frame in the media file */ + writePacket(AVMEDIA_TYPE_VIDEO, video_pkt); + } + // } + return image != null ? (video_pkt.flags() & AV_PKT_FLAG_KEY) != 0 : got_video_packet[0] != 0; + + } + } + + public boolean recordSamples(Buffer ... samples) throws Exception { + return recordSamples(0, 0, samples); + } + public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buffer ... samples) throws Exception { + try (PointerScope scope = new PointerScope()) { + + if (audio_st == null) { + throw new Exception("No audio output stream (Is audioChannels > 0 and has start() been called?)"); + } + if (!started) { + throw new Exception("start() was not called successfully!"); + } + + if (samples == null && samples_out[0].position() > 0) { + // Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2 + double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size); + writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor)); + return writeFrame((AVFrame)null); + } + + int ret; + + if (sampleRate <= 0) { + sampleRate = audio_c.sample_rate(); + } + if (audioChannels <= 0) { + audioChannels = audio_c.ch_layout().nb_channels(); + } + int inputSize = samples != null ? samples[0].limit() - samples[0].position() : 0; + int inputFormat = samples_format; + int inputChannels = samples != null && samples.length > 1 ? 1 : audioChannels; + int inputDepth = 0; + int outputFormat = audio_c.sample_fmt(); + int outputChannels = samples_out.length > 1 ? 1 : audio_c.ch_layout().nb_channels(); + int outputDepth = av_get_bytes_per_sample(outputFormat); + if (samples != null && samples[0] instanceof ByteBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_U8P : AV_SAMPLE_FMT_U8; + inputDepth = 1; + for (int i = 0; i < samples.length; i++) { + ByteBuffer b = (ByteBuffer)samples[i]; + if (samples_in[i] instanceof BytePointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((BytePointer)samples_in[i]).position(0).put(b.array(), b.position(), inputSize); + } else { + if (samples_in[i] != null) { + samples_in[i].releaseReference(); + } + samples_in[i] = new BytePointer(b).retainReference(); + } + } + } else if (samples != null && samples[0] instanceof ShortBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_S16P : AV_SAMPLE_FMT_S16; + inputDepth = 2; + for (int i = 0; i < samples.length; i++) { + ShortBuffer b = (ShortBuffer)samples[i]; + if (samples_in[i] instanceof ShortPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((ShortPointer)samples_in[i]).position(0).put(b.array(), samples[i].position(), inputSize); + } else { + if (samples_in[i] != null) { + samples_in[i].releaseReference(); + } + samples_in[i] = new ShortPointer(b).retainReference(); + } + } + } else if (samples != null && samples[0] instanceof IntBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_S32P : AV_SAMPLE_FMT_S32; + inputDepth = 4; + for (int i = 0; i < samples.length; i++) { + IntBuffer b = (IntBuffer)samples[i]; + if (samples_in[i] instanceof IntPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((IntPointer)samples_in[i]).position(0).put(b.array(), samples[i].position(), inputSize); + } else { + if (samples_in[i] != null) { + samples_in[i].releaseReference(); + } + samples_in[i] = new IntPointer(b).retainReference(); + } + } + } else if (samples != null && samples[0] instanceof FloatBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_FLTP : AV_SAMPLE_FMT_FLT; + inputDepth = 4; + for (int i = 0; i < samples.length; i++) { + FloatBuffer b = (FloatBuffer)samples[i]; + if (samples_in[i] instanceof FloatPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((FloatPointer)samples_in[i]).position(0).put(b.array(), b.position(), inputSize); + } else { + if (samples_in[i] != null) { + samples_in[i].releaseReference(); + } + samples_in[i] = new FloatPointer(b).retainReference(); + } + } + } else if (samples != null && samples[0] instanceof DoubleBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_DBLP : AV_SAMPLE_FMT_DBL; + inputDepth = 8; + for (int i = 0; i < samples.length; i++) { + DoubleBuffer b = (DoubleBuffer)samples[i]; + if (samples_in[i] instanceof DoublePointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((DoublePointer)samples_in[i]).position(0).put(b.array(), b.position(), inputSize); + } else { + if (samples_in[i] != null) { + samples_in[i].releaseReference(); + } + samples_in[i] = new DoublePointer(b).retainReference(); + } + } + } else if (samples != null) { + throw new Exception("Audio samples Buffer has unsupported type: " + samples); + } + + if (samples_convert_ctx == null || samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate) { + if (samples_convert_ctx == null) { + samples_convert_ctx = new SwrContext().retainReference(); + } + if ((ret = swr_alloc_set_opts2(samples_convert_ctx, audio_c.ch_layout(), outputFormat, audio_c.sample_rate(), + default_layout, inputFormat, sampleRate, 0, null)) < 0) { + throw new Exception("swr_alloc_set_opts2() error " + ret + ": Cannot allocate the conversion context."); + } else if ((ret = swr_init(samples_convert_ctx)) < 0) { + throw new Exception("swr_init() error " + ret + ": Cannot initialize the conversion context."); + } + samples_channels = audioChannels; + samples_format = inputFormat; + samples_rate = sampleRate; + } + + for (int i = 0; samples != null && i < samples.length; i++) { + samples_in[i].position(samples_in[i].position() * inputDepth). + limit((samples_in[i].position() + inputSize) * inputDepth); + } + while (true) { + int inputCount = (int)Math.min(samples != null ? (samples_in[0].limit() - samples_in[0].position()) / (inputChannels * inputDepth) : 0, Integer.MAX_VALUE); + int outputCount = (int)Math.min((samples_out[0].limit() - samples_out[0].position()) / (outputChannels * outputDepth), Integer.MAX_VALUE); + inputCount = Math.min(inputCount, (outputCount * sampleRate + audio_c.sample_rate() - 1) / audio_c.sample_rate()); + for (int i = 0; samples != null && i < samples.length; i++) { + plane_ptr.put(i, samples_in[i]); + } + for (int i = 0; i < samples_out.length; i++) { + plane_ptr2.put(i, samples_out[i]); + } + if ((ret = swr_convert(samples_convert_ctx, plane_ptr2, outputCount, plane_ptr, inputCount)) < 0) { + throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples."); + } else if (ret == 0) { + break; + } + for (int i = 0; samples != null && i < samples.length; i++) { + samples_in[i].position(samples_in[i].position() + inputCount * inputChannels * inputDepth); + } + for (int i = 0; i < samples_out.length; i++) { + samples_out[i].position(samples_out[i].position() + ret * outputChannels * outputDepth); + } + + if (samples == null || samples_out[0].position() >= samples_out[0].limit()) { + writeSamples(audio_input_frame_size); + } + } + return samples != null ? frame.key_frame() != 0 : writeFrame((AVFrame)null); + + } + } + + private void writeSamples(int nb_samples) throws Exception { + if (samples_out == null || samples_out.length == 0) { + return; + } + + frame.nb_samples(nb_samples); + avcodec_fill_audio_frame(frame, audio_c.ch_layout().nb_channels(), audio_c.sample_fmt(), samples_out[0], (int)samples_out[0].position(), 0); + for (int i = 0; i < samples_out.length; i++) { + int linesize = 0; + if (samples_out[0].position() > 0 && samples_out[0].position() < samples_out[0].limit()) { + // align the end of the buffer to a 32-byte boundary as sometimes required by FFmpeg + linesize = ((int)samples_out[i].position() + 31) & ~31; + } else { + linesize = (int)Math.min(samples_out[i].limit(), Integer.MAX_VALUE); + } + + frame.data(i, samples_out[i].position(0)); + frame.linesize(i, linesize); + } + frame.ch_layout(audio_c.ch_layout()); + frame.format(audio_c.sample_fmt()); + frame.quality(audio_c.global_quality()); + writeFrame(frame); + } + + private boolean writeFrame(AVFrame frame) throws Exception { + int ret; + + if ((ret = avcodec_send_frame(audio_c, frame)) < 0 && frame != null) { + throw new Exception("avcodec_send_frame() error " + ret + ": Error sending an audio frame for encoding."); + } + if (frame != null) { + frame.pts(frame.pts() + frame.nb_samples()); // magic required by libvorbis and webm + } + + /* if zero size, it means the image was buffered */ + got_audio_packet[0] = 0; + while (ret >= 0) { + av_new_packet(audio_pkt, audio_outbuf_size); + ret = avcodec_receive_packet(audio_c, audio_pkt); + if (ret == AVERROR_EAGAIN() || ret == AVERROR_EOF()) { + av_packet_unref(audio_pkt); + break; + } else if (ret < 0) { + av_packet_unref(audio_pkt); + throw new Exception("avcodec_receive_packet() error " + ret + ": Error during audio encoding."); + } + got_audio_packet[0] = 1; + + if (audio_pkt.pts() != AV_NOPTS_VALUE) { + audio_pkt.pts(av_rescale_q(audio_pkt.pts(), audio_c.time_base(), audio_st.time_base())); + } + if (audio_pkt.dts() != AV_NOPTS_VALUE) { + audio_pkt.dts(av_rescale_q(audio_pkt.dts(), audio_c.time_base(), audio_st.time_base())); + } + audio_pkt.flags(audio_pkt.flags() | AV_PKT_FLAG_KEY); + audio_pkt.stream_index(audio_st.index()); + + /* write the compressed frame in the media file */ + writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); + + if (frame == null) { + // avoid infinite loop with buggy codecs on flush + break; + } + } + + return got_audio_packet[0] != 0; + } + + private void writePacket(int mediaType, AVPacket avPacket) throws Exception { + AVStream avStream = (mediaType == AVMEDIA_TYPE_VIDEO) ? video_st : (mediaType == AVMEDIA_TYPE_AUDIO) ? audio_st : null; + String mediaTypeStr = (mediaType == AVMEDIA_TYPE_VIDEO) ? "video" : (mediaType == AVMEDIA_TYPE_AUDIO) ? "audio" : "unsupported media stream type"; + + synchronized (oc) { + int ret; + if (interleaved && avStream != null) { + if ((ret = av_interleaved_write_frame(oc, avPacket)) < 0) { + av_packet_unref(avPacket); + throw new Exception("av_interleaved_write_frame() error " + ret + " while writing interleaved " + mediaTypeStr + " packet."); + } + } else { + if ((ret = av_write_frame(oc, avPacket)) < 0) { + av_packet_unref(avPacket); + throw new Exception("av_write_frame() error " + ret + " while writing " + mediaTypeStr + " packet."); + } + } + } + av_packet_unref(avPacket); + } + + public synchronized boolean recordPacket(AVPacket pkt) throws Exception { + if (ifmt_ctx == null) { + throw new Exception("No input format context (Has start(AVFormatContext) been called?)"); + } + if (!started) { + throw new Exception("start() was not called successfully!"); + } + + if (pkt == null) { + return false; + } + + AVStream in_stream = ifmt_ctx.streams(pkt.stream_index()); + /** + * Repair the problem of error decoding and playback caused by the absence of dts/pts + * in the output audio/video file or audio/video stream, + * Comment out this line of code so that PTS / DTS can specify the timestamp manually. + */ + // pkt.dts(AV_NOPTS_VALUE); + // pkt.pts(AV_NOPTS_VALUE); + pkt.pos(-1); + if (in_stream.codecpar().codec_type() == AVMEDIA_TYPE_VIDEO && video_st != null) { + pkt.stream_index(video_st.index()); + pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), video_st.time_base())); + pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), video_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase pts calculation + pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), video_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase dts calculation + writePacket(AVMEDIA_TYPE_VIDEO, pkt); + } else if (in_stream.codecpar().codec_type() == AVMEDIA_TYPE_AUDIO && audio_st != null && (audioChannels > 0)) { + pkt.stream_index(audio_st.index()); + pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), audio_st.time_base())); + pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), audio_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase pts calculation + pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), audio_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase dts calculation + writePacket(AVMEDIA_TYPE_AUDIO, pkt); + } + + return true; + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/Frame.java b/src/main/java/org/bytedeco/javacv/Frame.java new file mode 100644 index 00000000..788c26e9 --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/Frame.java @@ -0,0 +1,380 @@ +/* + * Copyright (C) 2015-2021 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.DoubleBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.LongBuffer; +import java.nio.ShortBuffer; +import java.util.EnumSet; + +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.FloatPointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.LongPointer; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.ShortPointer; +import org.bytedeco.javacpp.indexer.ByteIndexer; +import org.bytedeco.javacpp.indexer.DoubleIndexer; +import org.bytedeco.javacpp.indexer.FloatIndexer; +import org.bytedeco.javacpp.indexer.Indexable; +import org.bytedeco.javacpp.indexer.Indexer; +import org.bytedeco.javacpp.indexer.IntIndexer; +import org.bytedeco.javacpp.indexer.LongIndexer; +import org.bytedeco.javacpp.indexer.ShortIndexer; +import org.bytedeco.javacpp.indexer.UByteIndexer; +import org.bytedeco.javacpp.indexer.UShortIndexer; + +/** + * A class to manage the data of audio and video frames. It it used by + * {@link CanvasFrame}, {@link FrameGrabber}, {@link FrameRecorder}, and their + * subclasses. We can also make the link with other APIs, such as Android, + * Java 2D, FFmpeg, and OpenCV, via a {@link FrameConverter}. + * + * @author Samuel Audet + */ +public class Frame implements AutoCloseable, Indexable { + /** A flag set by a FrameGrabber or a FrameRecorder to indicate a key frame. */ + public boolean keyFrame; + + /** The type of the image frame ('I', 'P', 'B', etc). */ + public char pictType; + + /** Constants to be used for {@link #imageDepth}. */ + public static final int + DEPTH_BYTE = -8, + DEPTH_UBYTE = 8, + DEPTH_SHORT = -16, + DEPTH_USHORT = 16, + DEPTH_INT = -32, + DEPTH_LONG = -64, + DEPTH_FLOAT = 32, + DEPTH_DOUBLE = 64; + + /** Constants defining data type in the frame. */ + public static enum Type { + VIDEO, + AUDIO, + DATA, + SUBTITLE, + ATTACHMENT + } + + /** Information associated with the {@link #image} field. */ + public int imageWidth, imageHeight, imageDepth, imageChannels, imageStride; + + /** + * Buffers to hold image pixels from multiple channels for a video frame. + * Most of the software supports packed data only, but an array is provided + * to allow users to store images in a planar format as well. + */ + public Buffer[] image; + + /** Information associated with the {@link #samples} field. */ + public int sampleRate, audioChannels; + + /** Buffers to hold audio samples from multiple channels for an audio frame. */ + public Buffer[] samples; + + /** Buffer to hold a data stream associated with a frame. */ + public ByteBuffer data; + + /** Stream number the audio|video|other data is associated with. */ + public int streamIndex; + + /** The type of the stream. */ + public Type type; + + /** The underlying data object, for example, Pointer, AVFrame, IplImage, or Mat. */ + public Object opaque; + + /** Timestamp of the frame creation in microseconds. */ + public long timestamp; + + /** Returns {@code Math.abs(depth) / 8}. */ + public static int pixelSize(int depth) { + return Math.abs(depth) / 8; + } + + /** Empty constructor. */ + public Frame() { } + + /** Allocates a new packed image frame in native memory where rows are 8-byte aligned. */ + public Frame(int width, int height, int depth, int channels) { + this(width, height, depth, channels, ((width * channels * pixelSize(depth) + 7) & ~7) / pixelSize(depth)); + } + public Frame(int width, int height, int depth, int channels, int imageStride) { + this.imageWidth = width; + this.imageHeight = height; + this.imageDepth = depth; + this.imageChannels = channels; + this.imageStride = imageStride; + this.pictType = '\0'; + this.image = new Buffer[1]; + this.data = null; + this.streamIndex = -1; + this.type = null; + + Pointer pointer = new BytePointer(imageHeight * imageStride * pixelSize(depth)); + ByteBuffer buffer = pointer.asByteBuffer(); + switch (imageDepth) { + case DEPTH_BYTE: + case DEPTH_UBYTE: image[0] = buffer; break; + case DEPTH_SHORT: + case DEPTH_USHORT: image[0] = buffer.asShortBuffer(); break; + case DEPTH_INT: image[0] = buffer.asIntBuffer(); break; + case DEPTH_LONG: image[0] = buffer.asLongBuffer(); break; + case DEPTH_FLOAT: image[0] = buffer.asFloatBuffer(); break; + case DEPTH_DOUBLE: image[0] = buffer.asDoubleBuffer(); break; + default: throw new UnsupportedOperationException("Unsupported depth value: " + imageDepth); + } + opaque = new Pointer[] {pointer.retainReference()}; + } + + /** Returns {@code createIndexer(true, 0)}. */ + public I createIndexer() { + return (I)createIndexer(true, 0); + } + @Override public I createIndexer(boolean direct) { + return (I)createIndexer(direct, 0); + } + /** Returns an {@link Indexer} for the ith image plane. */ + public I createIndexer(boolean direct, int i) { + long[] sizes = {imageHeight, imageWidth, imageChannels}; + long[] strides = {imageStride, imageChannels, 1}; + Buffer buffer = image[i]; + Object array = buffer.hasArray() ? buffer.array() : null; + switch (imageDepth) { + case DEPTH_UBYTE: + return array != null ? (I)UByteIndexer.create((byte[])array, sizes, strides).indexable(this) + : direct ? (I)UByteIndexer.create((ByteBuffer)buffer, sizes, strides).indexable(this) + : (I)UByteIndexer.create(new BytePointer((ByteBuffer)buffer), sizes, strides, false).indexable(this); + case DEPTH_BYTE: + return array != null ? (I)ByteIndexer.create((byte[])array, sizes, strides).indexable(this) + : direct ? (I)ByteIndexer.create((ByteBuffer)buffer, sizes, strides).indexable(this) + : (I)ByteIndexer.create(new BytePointer((ByteBuffer)buffer), sizes, strides, false).indexable(this); + case DEPTH_USHORT: + return array != null ? (I)UShortIndexer.create((short[])array, sizes, strides).indexable(this) + : direct ? (I)UShortIndexer.create((ShortBuffer)buffer, sizes, strides).indexable(this) + : (I)UShortIndexer.create(new ShortPointer((ShortBuffer)buffer), sizes, strides, false).indexable(this); + case DEPTH_SHORT: + return array != null ? (I)ShortIndexer.create((short[])array, sizes, strides).indexable(this) + : direct ? (I)ShortIndexer.create((ShortBuffer)buffer, sizes, strides).indexable(this) + : (I)ShortIndexer.create(new ShortPointer((ShortBuffer)buffer), sizes, strides, false).indexable(this); + case DEPTH_INT: + return array != null ? (I)IntIndexer.create((int[])array, sizes, strides).indexable(this) + : direct ? (I)IntIndexer.create((IntBuffer)buffer, sizes, strides).indexable(this) + : (I)IntIndexer.create(new IntPointer((IntBuffer)buffer), sizes, strides, false).indexable(this); + case DEPTH_LONG: + return array != null ? (I)LongIndexer.create((long[])array, sizes, strides).indexable(this) + : direct ? (I)LongIndexer.create((LongBuffer)buffer, sizes, strides).indexable(this) + : (I)LongIndexer.create(new LongPointer((LongBuffer)buffer), sizes, strides, false).indexable(this); + case DEPTH_FLOAT: + return array != null ? (I)FloatIndexer.create((float[])array, sizes, strides).indexable(this) + : direct ? (I)FloatIndexer.create((FloatBuffer)buffer, sizes, strides).indexable(this) + : (I)FloatIndexer.create(new FloatPointer((FloatBuffer)buffer), sizes, strides, false).indexable(this); + case DEPTH_DOUBLE: + return array != null ? (I)DoubleIndexer.create((double[])array, sizes, strides).indexable(this) + : direct ? (I)DoubleIndexer.create((DoubleBuffer)buffer, sizes, strides).indexable(this) + : (I)DoubleIndexer.create(new DoublePointer((DoubleBuffer)buffer), sizes, strides, false).indexable(this); + default: assert false; + } + return null; + } + + /**Care must be taken if this method is to be used in conjunction with movie recordings. + * Cloning a frame containing a full HD picture (alpha channel included) would take 1920 x 1080 * 4 = 8.294.400 Bytes. + * Expect a heap overflow exception when using this method without cleaning up. + * + * @return A deep copy of this frame. + * @see {@link #cloneBufferArray} + * + * Extension proposed by Dragos Dutu + * */ + @Override + public Frame clone() { + Frame newFrame = new Frame(); + + // Video part + newFrame.imageWidth = imageWidth; + newFrame.imageHeight = imageHeight; + newFrame.imageDepth = imageDepth; + newFrame.imageChannels = imageChannels; + newFrame.imageStride = imageStride; + newFrame.keyFrame = keyFrame; + newFrame.pictType = pictType; + newFrame.streamIndex = streamIndex; + newFrame.type = type; + newFrame.opaque = new Pointer[3]; + if (image != null) { + newFrame.image = new Buffer[image.length]; + ((Pointer[])newFrame.opaque)[0] = cloneBufferArray(image, newFrame.image); + } + + // Audio part + newFrame.audioChannels = audioChannels; + newFrame.sampleRate = sampleRate; + if (samples != null) { + newFrame.samples = new Buffer[samples.length]; + ((Pointer[])newFrame.opaque)[1] = cloneBufferArray(samples, newFrame.samples); + } + + // Other data streams + if (data != null) { + ByteBuffer[] dst = new ByteBuffer[1]; + ((Pointer[])newFrame.opaque)[2] = cloneBufferArray(new ByteBuffer[]{data}, dst); + newFrame.data = dst[0]; + } + + // Add timestamp + newFrame.timestamp = timestamp; + + return newFrame; + } + + /** + * This private method takes a buffer array as input and returns a deep copy. + * It is assumed that all buffers in the input array are of the same subclass. + * + * @param srcBuffers - Buffer array to be cloned + * @param clonedBuffers - Buffer array to fill with clones + * @return Opaque object to store + * + * @author Extension proposed by Dragos Dutu + */ + private static Pointer cloneBufferArray(Buffer[] srcBuffers, Buffer[] clonedBuffers) { + Pointer opaque = null; + + if (srcBuffers != null && srcBuffers.length > 0) { + int totalCapacity = 0; + for (int i = 0; i < srcBuffers.length; i++) { + srcBuffers[i].rewind(); + totalCapacity += srcBuffers[i].capacity(); + } + + /* + * In order to optimize the transfer we need a type check. + * + * Most CPUs support hardware memory transfer for different data + * types, so it's faster to copy more bytes at once rather + * than one byte per iteration as in case of ByteBuffer. + * + * For example, Intel CPUs support MOVSB (byte transfer), MOVSW + * (word transfer), MOVSD (double word transfer), MOVSS (32 bit + * scalar single precision floating point), MOVSQ (quad word + * transfer) and so on... + * + * Type checking may be improved by changing the order in + * which a buffer is checked against. If it's likely that the + * expected buffer is of type "ShortBuffer", then it should be + * checked at first place. + * + */ + + if (srcBuffers[0] instanceof ByteBuffer) { + BytePointer pointer = new BytePointer(totalCapacity); + for (int i = 0; i < srcBuffers.length; i++) { + clonedBuffers[i] = pointer.limit(pointer.position() + srcBuffers[i].limit()) + .asBuffer().put((ByteBuffer)srcBuffers[i]); + pointer.position(pointer.limit()); + } + opaque = pointer; + } else if (srcBuffers[0] instanceof ShortBuffer) { + ShortPointer pointer = new ShortPointer(totalCapacity); + for (int i = 0; i < srcBuffers.length; i++) { + clonedBuffers[i] = pointer.limit(pointer.position() + srcBuffers[i].limit()) + .asBuffer().put((ShortBuffer)srcBuffers[i]); + pointer.position(pointer.limit()); + } + opaque = pointer; + } else if (srcBuffers[0] instanceof IntBuffer) { + IntPointer pointer = new IntPointer(totalCapacity); + for (int i = 0; i < srcBuffers.length; i++) { + clonedBuffers[i] = pointer.limit(pointer.position() + srcBuffers[i].limit()) + .asBuffer().put((IntBuffer)srcBuffers[i]); + pointer.position(pointer.limit()); + } + opaque = pointer; + } else if (srcBuffers[0] instanceof LongBuffer) { + LongPointer pointer = new LongPointer(totalCapacity); + for (int i = 0; i < srcBuffers.length; i++) { + clonedBuffers[i] = pointer.limit(pointer.position() + srcBuffers[i].limit()) + .asBuffer().put((LongBuffer)srcBuffers[i]); + pointer.position(pointer.limit()); + } + opaque = pointer; + } else if (srcBuffers[0] instanceof FloatBuffer) { + FloatPointer pointer = new FloatPointer(totalCapacity); + for (int i = 0; i < srcBuffers.length; i++) { + clonedBuffers[i] = pointer.limit(pointer.position() + srcBuffers[i].limit()) + .asBuffer().put((FloatBuffer)srcBuffers[i]); + pointer.position(pointer.limit()); + } + opaque = pointer; + } else if (srcBuffers[0] instanceof DoubleBuffer) { + DoublePointer pointer = new DoublePointer(totalCapacity); + for (int i = 0; i < srcBuffers.length; i++) { + clonedBuffers[i] = pointer.limit(pointer.position() + srcBuffers[i].limit()) + .asBuffer().put((DoubleBuffer)srcBuffers[i]); + pointer.position(pointer.limit()); + } + opaque = pointer; + } + + for (int i = 0; i < srcBuffers.length; i++) { + srcBuffers[i].rewind(); + clonedBuffers[i].rewind(); + } + } + + if (opaque != null) { + opaque.retainReference(); + } + return opaque; + } + + /** Returns types of data containing in the frame */ + public EnumSet getTypes() { + EnumSet type = EnumSet.noneOf(Type.class); + if (image != null) type.add(Type.VIDEO); + if (samples != null) type.add(Type.AUDIO); + if (data != null) type.add(Type.DATA); + return type; + } + + @Override public void close() { + if (opaque instanceof Pointer[]) { + for (Pointer p : (Pointer[])opaque) { + if (p != null) { + p.releaseReference(); + p = null; + } + } + opaque = null; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/FrameConverter.java b/src/main/java/org/bytedeco/javacv/FrameConverter.java new file mode 100644 index 00000000..51ae8a2b --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/FrameConverter.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2015-2021 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +/** + * Defines two methods to convert between a {@link Frame} and another generic + * data object that can contain the same data. The idea with this design is + * to allow users to convert easily between multiple potentially mutually + * exclusive types of image data objects over which we have no control. Because + * of this, and for performance reasons, any object returned by this class is + * guaranteed to remain valid only until the next call to {@code convert()}, + * anywhere in a chain of {@code FrameConverter} objects, and only as long as + * the latter themselves are not closed or garbage collected. + * + * @author Samuel Audet + */ +public abstract class FrameConverter implements AutoCloseable { + protected Frame frame; + + public abstract Frame convert(F f); + public abstract F convert(Frame frame); + + @Override public void close() { + if (frame != null) { + frame.close(); + frame = null; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/FrameGrabber.java b/src/main/java/org/bytedeco/javacv/FrameGrabber.java new file mode 100644 index 00000000..eff55aa4 --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/FrameGrabber.java @@ -0,0 +1,810 @@ +/* + * Copyright (C) 2009-2022 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +import java.beans.PropertyEditorSupport; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.Buffer; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * + * @author Samuel Audet + */ +public abstract class FrameGrabber implements Closeable { + + public static final List list = new LinkedList(Arrays.asList(new String[] { + "DC1394", "FlyCapture", "FlyCapture2", "OpenKinect", "OpenKinect2", "RealSense", "RealSense2", "PS3Eye", "VideoInput", "OpenCV", "FFmpeg", "IPCamera" })); + public static void init() { + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + } catch (Throwable t) { + continue; + } + } + } + public static Class getDefault() { + // select first frame grabber that can load and that may have some cameras.. + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + boolean mayContainCameras = false; + try { + String[] s = (String[])c.getMethod("getDeviceDescriptions").invoke(null); + if (s.length > 0) { + mayContainCameras = true; + } + } catch (Throwable t) { + if (t.getCause() instanceof UnsupportedOperationException) { + mayContainCameras = true; + } + } + if (mayContainCameras) { + return c; + } + } catch (Throwable t) { + continue; + } + } + return null; + } + public static Class get(String className) throws Exception { + className = FrameGrabber.class.getPackage().getName() + "." + className; + try { + return Class.forName(className).asSubclass(FrameGrabber.class); + } catch (ClassNotFoundException e) { + String className2 = className + "FrameGrabber"; + try { + return Class.forName(className2).asSubclass(FrameGrabber.class); + } catch (ClassNotFoundException ex) { + throw new Exception("Could not get FrameGrabber class for " + className + " or " + className2, e); + } + } + } + + public static FrameGrabber create(Class c, Class p, Object o) throws Exception { + Throwable cause = null; + try { + return c.getConstructor(p).newInstance(o); + } catch (InstantiationException ex) { + cause = ex; + } catch (IllegalAccessException ex) { + cause = ex; + } catch (IllegalArgumentException ex) { + cause = ex; + } catch (NoSuchMethodException ex) { + cause = ex; + } catch (InvocationTargetException ex) { + cause = ex.getCause(); + } + throw new Exception("Could not create new " + c.getSimpleName() + "(" + o + ")", cause); + } + + public static FrameGrabber createDefault(File deviceFile) throws Exception { + return create(getDefault(), File.class, deviceFile); + } + public static FrameGrabber createDefault(String devicePath) throws Exception { + return create(getDefault(), String.class, devicePath); + } + public static FrameGrabber createDefault(int deviceNumber) throws Exception { + try { + return create(getDefault(), int.class, deviceNumber); + } catch (Exception ex) { + return create(getDefault(), Integer.class, deviceNumber); + } + } + + public static FrameGrabber create(String className, File deviceFile) throws Exception { + return create(get(className), File.class, deviceFile); + } + public static FrameGrabber create(String className, String devicePath) throws Exception { + return create(get(className), String.class, devicePath); + } + public static FrameGrabber create(String className, int deviceNumber) throws Exception { + try { + return create(get(className), int.class, deviceNumber); + } catch (Exception ex) { + return create(get(className), Integer.class, deviceNumber); + } + } + + public static class PropertyEditor extends PropertyEditorSupport { + @Override public String getAsText() { + Class c = (Class)getValue(); + return c == null ? "null" : c.getSimpleName().split("FrameGrabber")[0]; + } + @Override public void setAsText(String s) { + if (s == null) { + setValue(null); + } + try { + setValue(get(s)); + } catch (Exception ex) { + throw new IllegalArgumentException(ex); + } + } + @Override public String[] getTags() { + return list.toArray(new String[list.size()]); + } + } + + + public static enum ImageMode { + COLOR, GRAY, RAW + } + + public static enum SampleMode { + SHORT, FLOAT, RAW + } + + public static final long + SENSOR_PATTERN_RGGB = 0, + SENSOR_PATTERN_GBRG = (1L << 32), + SENSOR_PATTERN_GRBG = 1, + SENSOR_PATTERN_BGGR = (1L << 32) | 1; + + protected int videoStream = -1, audioStream = -1; + protected int videoDisposition = 0, audioDisposition = 0; + protected String format = null, videoCodecName = null, audioCodecName = null; + protected int imageWidth = 0, imageHeight = 0, audioChannels = 0; + protected ImageMode imageMode = ImageMode.COLOR; + protected long sensorPattern = -1L; + protected int pixelFormat = -1, videoCodec, videoBitrate = 0, imageScalingFlags = 0; + protected double aspectRatio = 0, frameRate = 0; + protected SampleMode sampleMode = SampleMode.SHORT; + protected int sampleFormat = -1, audioCodec, audioBitrate = 0, sampleRate = 0; + protected boolean triggerMode = false; + protected int bpp = 0; + protected int timeout = 10000; + protected int numBuffers = 4; + protected double gamma = 0.0; + protected boolean deinterlace = false; + protected Charset charset = Charset.defaultCharset(); + protected Map options = new HashMap(); + protected Map videoOptions = new HashMap(); + protected Map audioOptions = new HashMap(); + protected Map metadata = new HashMap(); + protected Map videoMetadata = new HashMap(); + protected Map audioMetadata = new HashMap(); + protected Map videoSideData = new HashMap(); + protected Map audioSideData = new HashMap(); + protected int frameNumber = 0; + protected long timestamp = 0; + protected int maxDelay = -1; + protected long startTime = 0; + + public int getVideoStream() { + return videoStream; + } + public void setVideoStream(int videoStream) { + this.videoStream = videoStream; + } + + public int getAudioStream() { + return audioStream; + } + public void setAudioStream(int audioStream) { + this.audioStream = audioStream; + } + + public void setVideoDisposition(int videoDisposition) { + this.videoDisposition = videoDisposition; + } + public int getVideoDisposition() { + return videoDisposition; + } + + public void setAudioDisposition(int audioDisposition) { + this.audioDisposition = audioDisposition; + } + public int getAudioDisposition() { + return audioDisposition; + } + + public String getFormat() { + return format; + } + public void setFormat(String format) { + this.format = format; + } + + public String getVideoCodecName() { + return videoCodecName; + } + public void setVideoCodecName(String videoCodecName) { + this.videoCodecName = videoCodecName; + } + + public String getAudioCodecName() { + return audioCodecName; + } + public void setAudioCodecName(String audioCodecName) { + this.audioCodecName = audioCodecName; + } + + public int getImageWidth() { + return imageWidth; + } + public void setImageWidth(int imageWidth) { + this.imageWidth = imageWidth; + } + + public int getImageHeight() { + return imageHeight; + } + public void setImageHeight(int imageHeight) { + this.imageHeight = imageHeight; + } + + public int getAudioChannels() { + return audioChannels; + } + public void setAudioChannels(int audioChannels) { + this.audioChannels = audioChannels; + } + + public ImageMode getImageMode() { + return imageMode; + } + public void setImageMode(ImageMode imageMode) { + this.imageMode = imageMode; + } + + public long getSensorPattern() { + return sensorPattern; + } + public void setSensorPattern(long sensorPattern) { + this.sensorPattern = sensorPattern; + } + + public int getPixelFormat() { + return pixelFormat; + } + public void setPixelFormat(int pixelFormat) { + this.pixelFormat = pixelFormat; + } + + public int getVideoCodec() { + return videoCodec; + } + public void setVideoCodec(int videoCodec) { + this.videoCodec = videoCodec; + } + + public int getVideoBitrate() { + return videoBitrate; + } + public void setVideoBitrate(int videoBitrate) { + this.videoBitrate = videoBitrate; + } + + public int getImageScalingFlags() { + return imageScalingFlags; + } + public void setImageScalingFlags(int imageScalingFlags) { + this.imageScalingFlags = imageScalingFlags; + } + + public double getAspectRatio() { + return aspectRatio; + } + public void setAspectRatio(double aspectRatio) { + this.aspectRatio = aspectRatio; + } + + public double getFrameRate() { + return frameRate; + } + public void setFrameRate(double frameRate) { + this.frameRate = frameRate; + } + + public int getAudioCodec() { + return audioCodec; + } + public void setAudioCodec(int audioCodec) { + this.audioCodec = audioCodec; + } + + public int getAudioBitrate() { + return audioBitrate; + } + public void setAudioBitrate(int audioBitrate) { + this.audioBitrate = audioBitrate; + } + + public SampleMode getSampleMode() { + return sampleMode; + } + public void setSampleMode(SampleMode samplesMode) { + this.sampleMode = samplesMode; + } + + public int getSampleFormat() { + return sampleFormat; + } + public void setSampleFormat(int sampleFormat) { + this.sampleFormat = sampleFormat; + } + + public int getSampleRate() { + return sampleRate; + } + public void setSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + } + + public boolean isTriggerMode() { + return triggerMode; + } + public void setTriggerMode(boolean triggerMode) { + this.triggerMode = triggerMode; + } + + public int getBitsPerPixel() { + return bpp; + } + public void setBitsPerPixel(int bitsPerPixel) { + this.bpp = bitsPerPixel; + } + + public int getTimeout() { + return timeout; + } + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public int getNumBuffers() { + return numBuffers; + } + public void setNumBuffers(int numBuffers) { + this.numBuffers = numBuffers; + } + + public double getGamma() { + return gamma; + } + public void setGamma(double gamma) { + this.gamma = gamma; + } + + public boolean isDeinterlace() { + return deinterlace; + } + public void setDeinterlace(boolean deinterlace) { + this.deinterlace = deinterlace; + } + + public Charset getCharset() { + return charset; + } + public void setCharset(Charset charset) { + this.charset = charset; + } + + public Map getOptions() { + return options; + } + public void setOptions(Map options) { + this.options = options; + } + + public Map getVideoOptions() { + return videoOptions; + } + public void setVideoOptions(Map options) { + this.videoOptions = options; + } + + public Map getAudioOptions() { + return audioOptions; + } + public void setAudioOptions(Map options) { + this.audioOptions = options; + } + + public Map getMetadata() { + return metadata; + } + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public Map getVideoMetadata() { + return videoMetadata; + } + public void setVideoMetadata(Map metadata) { + this.videoMetadata = metadata; + } + + public Map getAudioMetadata() { + return audioMetadata; + } + public void setAudioMetadata(Map metadata) { + this.audioMetadata = metadata; + } + + public String getOption(String key) { + return options.get(key); + } + public void setOption(String key, String value) { + options.put(key, value); + } + + public String getVideoOption(String key) { + return videoOptions.get(key); + } + public void setVideoOption(String key, String value) { + videoOptions.put(key, value); + } + + public String getAudioOption(String key) { + return audioOptions.get(key); + } + public void setAudioOption(String key, String value) { + audioOptions.put(key, value); + } + + public String getMetadata(String key) { + return metadata.get(key); + } + public void setMetadata(String key, String value) { + metadata.put(key, value); + } + + public String getVideoMetadata(String key) { + return videoMetadata.get(key); + } + public void setVideoMetadata(String key, String value) { + videoMetadata.put(key, value); + } + + public String getAudioMetadata(String key) { + return audioMetadata.get(key); + } + public void setAudioMetadata(String key, String value) { + audioMetadata.put(key, value); + } + + public Map getVideoSideData() { + return videoSideData; + } + public void setVideoSideData(Map videoSideData) { + this.videoSideData = videoSideData; + } + + public Buffer getVideoSideData(String key) { + return videoSideData.get(key); + } + public void setVideoSideData(String key, Buffer value) { + videoSideData.put(key, value); + } + + public Map getAudioSideData() { + return audioSideData; + } + public void setAudioSideData(Map audioSideData) { + this.audioSideData = audioSideData; + } + + public Buffer getAudioSideData(String key) { + return audioSideData.get(key); + } + public void setAudioSideData(String key, Buffer value) { + audioSideData.put(key, value); + } + + public int getFrameNumber() { + return frameNumber; + } + public void setFrameNumber(int frameNumber) throws Exception { + this.frameNumber = frameNumber; + } + + public long getTimestamp() { + return timestamp; + } + public void setTimestamp(long timestamp) throws Exception { + this.timestamp = timestamp; + } + + public int getMaxDelay() { + return maxDelay; + } + public void setMaxDelay(int maxDelay) { + this.maxDelay = maxDelay; + } + + public int getLengthInFrames() { + return 0; + } + public long getLengthInTime() { + return 0; + } + + public static class Exception extends IOException { + public Exception(String message) { super(message); } + public Exception(String message, Throwable cause) { super(message, cause); } + } + + public abstract void start() throws Exception; + public abstract void stop() throws Exception; + public abstract void trigger() throws Exception; + + @Override public void close() throws Exception { + stop(); + release(); + } + + /** + * Each call to grab stores the new image in the memory address for the previously returned frame.
+ * IE.
+ * + * grabber.grab() == grabber.grab() + * + *
+ * This means that if you need to cache images returned from grab you should {@link Frame#clone()} the + * returned frame as the next call to grab will overwrite your existing image's memory. + *
+ * Why?
+ * Using this method instead of allocating a new buffer every time a frame + * is grabbed improves performance by reducing the frequency of garbage collections. + * Almost no additional heap space is typically allocated per frame. + * + * @return The frame returned from the grabber + * @throws Exception If there is a problem grabbing the frame. + */ + public abstract Frame grab() throws Exception; + public Frame grabFrame() throws Exception { return grab(); } + public abstract void release() throws Exception; + + public void restart() throws Exception { + stop(); + start(); + } + public void flush() throws Exception { + for (int i = 0; i < numBuffers+1; i++) { + grab(); + } + } + + private ExecutorService executor = Executors.newSingleThreadExecutor(); + private Future future = null; + private Frame delayedFrame = null; + private long delayedTime = 0; + public void delayedGrab(final long delayTime) { + delayedFrame = null; + delayedTime = 0; + final long start = System.nanoTime()/1000; + if (future != null && !future.isDone()) { + return; + } + future = executor.submit(new Callable() { public Void call() throws Exception { + do { + delayedFrame = grab(); + delayedTime = System.nanoTime()/1000 - start; + } while (delayedTime < delayTime); + return null; + }}); + } + public long getDelayedTime() throws InterruptedException, ExecutionException { + if (future == null) { + return 0; + } + future.get(); + return delayedTime; + } + public Frame getDelayedFrame() throws InterruptedException, ExecutionException { + if (future == null) { + return null; + } + future.get(); + return delayedFrame; + } + + public static class Array { + // declared protected to force users to use createArray(), which + // can be overridden without changing the calling code... + protected Array(FrameGrabber[] frameGrabbers) { + setFrameGrabbers(frameGrabbers); + } + + private Frame[] grabbedFrames = null; + private long[] latencies = null; + private long[] bestLatencies = null; + private long lastNewestTimestamp = 0; + private long bestInterval = Long.MAX_VALUE; + + protected FrameGrabber[] frameGrabbers = null; + public FrameGrabber[] getFrameGrabbers() { + return frameGrabbers; + } + public void setFrameGrabbers(FrameGrabber[] frameGrabbers) { + this.frameGrabbers = frameGrabbers; + grabbedFrames = new Frame[frameGrabbers.length]; + latencies = new long[frameGrabbers.length]; + bestLatencies = null; + lastNewestTimestamp = 0; + } + public int size() { + return frameGrabbers.length; + } + + public void start() throws Exception { + for (FrameGrabber f : frameGrabbers) { + f.start(); + } + } + public void stop() throws Exception { + for (FrameGrabber f : frameGrabbers) { + f.stop(); + } + } + // should be overriden to implement a broadcast trigger... + public void trigger() throws Exception { + for (FrameGrabber f : frameGrabbers) { + if (f.isTriggerMode()) { + f.trigger(); + } + } + } + // should be overriden to implement a broadcast grab... + public Frame[] grab() throws Exception { + if (frameGrabbers.length == 1) { + grabbedFrames[0] = frameGrabbers[0].grab(); + return grabbedFrames; + } + + // assume we sometimes get perfectly synchronized images, + // so save the best latencies we find as the perfectly + // synchronized case, so we know what to aim for in + // cases of missing/dropped frames ... + long newestTimestamp = 0; + boolean unsynchronized = false; + for (int i = 0; i < frameGrabbers.length; i++) { + grabbedFrames[i] = frameGrabbers[i].grab(); + if (grabbedFrames[i] != null) { + newestTimestamp = Math.max(newestTimestamp, frameGrabbers[i].getTimestamp()); + } + if (frameGrabbers[i].getClass() != frameGrabbers[(i + 1) % frameGrabbers.length].getClass()) { + // assume we can't synchronize different types of cameras with each other + unsynchronized = true; + } + } + if (unsynchronized) { + return grabbedFrames; + } + for (int i = 0; i < frameGrabbers.length; i++) { + if (grabbedFrames[i] != null) { + latencies[i] = newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp()); + } + } + if (bestLatencies == null) { + bestLatencies = Arrays.copyOf(latencies, latencies.length); + } else { + int sum1 = 0, sum2 = 0; + for (int i = 0; i < frameGrabbers.length; i++) { + sum1 += latencies[i]; + sum2 += bestLatencies[i]; + } + if (sum1 < sum2) { + bestLatencies = Arrays.copyOf(latencies, latencies.length); + } + } + + // we cannot have latencies higher than the time between frames.. + // or something too close to it anyway... 90% is good? + bestInterval = Math.min(bestInterval, newestTimestamp-lastNewestTimestamp); + for (int i = 0; i < bestLatencies.length; i++) { + bestLatencies[i] = Math.min(bestLatencies[i], bestInterval*9/10); + } + + // try to synchronize by attempting to land within 10% of + // the bestLatencies looking up to 2 frames ahead ... + for (int j = 0; j < 2; j++) { + for (int i = 0; i < frameGrabbers.length; i++) { + if (frameGrabbers[i].isTriggerMode() || grabbedFrames[i] == null) { + continue; + } + int latency = (int)(newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp())); + while (latency-bestLatencies[i] > 0.1*bestLatencies[i]) { + grabbedFrames[i] = frameGrabbers[i].grab(); + if (grabbedFrames[i] == null) { + break; + } + latency = (int)(newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp())); + if (latency < 0) { + // woops, a camera seems to have dropped a frame somewhere... + // bump up the newestTimestamp + newestTimestamp = Math.max(0, frameGrabbers[i].getTimestamp()); + break; + } + } + } + } + + //for (int i = 0; i < frameGrabbers.length; i++) { + // long latency = newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp()); + // System.out.print(bestLatencies[i] + " " + latency + " "); + //} + //System.out.println(" " + bestInterval); + + lastNewestTimestamp = newestTimestamp; + + return grabbedFrames; + } + public void release() throws Exception { + for (FrameGrabber f : frameGrabbers) { + f.release(); + } + } + } + + public Array createArray(FrameGrabber[] frameGrabbers) { + return new Array(frameGrabbers); + } + + /** Returns {@code frame = grab()} after {@code waitForTimestamp(frame)}. */ + public Frame grabAtFrameRate() throws Exception, InterruptedException { + Frame frame = grab(); + if (frame != null) { + waitForTimestamp(frame); + } + return frame; + } + + /** Returns true if {@code Thread.sleep()} had to be called. */ + public boolean waitForTimestamp(Frame frame) throws InterruptedException { + if (startTime == 0) { + startTime = System.nanoTime() / 1000 - frame.timestamp; + } else { + long delay = frame.timestamp - (System.nanoTime() / 1000 - startTime); + if (delay > 0) { + Thread.sleep(delay / 1000, (int)(delay % 1000) * 1000); + return true; + } + } + return false; + } + + public void resetStartTime() { + startTime = 0; + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/FrameRecorder.java b/src/main/java/org/bytedeco/javacv/FrameRecorder.java new file mode 100644 index 00000000..04f2da6b --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/FrameRecorder.java @@ -0,0 +1,442 @@ +/* + * Copyright (C) 2009-2023 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.Buffer; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * + * @author Samuel Audet + */ +public abstract class FrameRecorder implements Closeable { + + public static final List list = new LinkedList(Arrays.asList(new String[] { "FFmpeg", "OpenCV" })); + public static void init() { + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + } catch (Throwable t) { } + } + } + public static Class getDefault() { + // select first frame recorder that can load.. + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + return c; + } catch (Throwable t) { } + } + return null; + } + public static Class get(String className) throws Exception { + className = FrameRecorder.class.getPackage().getName() + "." + className; + try { + return Class.forName(className).asSubclass(FrameRecorder.class); + } catch (ClassNotFoundException e) { + String className2 = className + "FrameRecorder"; + try { + return Class.forName(className2).asSubclass(FrameRecorder.class); + } catch (ClassNotFoundException ex) { + throw new Exception("Could not get FrameRecorder class for " + className + " or " + className2, e); + } + } + } + + public static FrameRecorder create(Class c, Class p, Object o, int w, int h) throws Exception { + Throwable cause = null; + try { + return (FrameRecorder)c.getConstructor(p, int.class, int.class).newInstance(o, w, h); + } catch (InstantiationException ex) { + cause = ex; + } catch (IllegalAccessException ex) { + cause = ex; + } catch (IllegalArgumentException ex) { + cause = ex; + } catch (NoSuchMethodException ex) { + cause = ex; + } catch (InvocationTargetException ex) { + cause = ex.getCause(); + } + throw new Exception("Could not create new " + c.getSimpleName() + "(" + o + ", " + w + ", " + h + ")", cause); + } + + public static FrameRecorder createDefault(File file, int width, int height) throws Exception { + return create(getDefault(), File.class, file, width, height); + } + public static FrameRecorder createDefault(String filename, int width, int height) throws Exception { + return create(getDefault(), String.class, filename, width, height); + } + + public static FrameRecorder create(String className, File file, int width, int height) throws Exception { + return create(get(className), File.class, file, width, height); + } + public static FrameRecorder create(String className, String filename, int width, int height) throws Exception { + return create(get(className), String.class, filename, width, height); + } + + protected String format, videoCodecName, audioCodecName; + protected int imageWidth, imageHeight, audioChannels; + protected int pixelFormat, videoCodec, videoBitrate, imageScalingFlags, gopSize = -1; + protected double aspectRatio, frameRate, videoQuality = -1; + protected int sampleFormat, audioCodec, audioBitrate, sampleRate; + protected double audioQuality = -1; + protected boolean interleaved; + protected Charset charset = Charset.defaultCharset(); + protected Map options = new HashMap(); + protected Map videoOptions = new HashMap(); + protected Map audioOptions = new HashMap(); + protected Map metadata = new HashMap(); + protected Map videoMetadata = new HashMap(); + protected Map audioMetadata = new HashMap(); + protected Map videoSideData = new HashMap(); + protected Map audioSideData = new HashMap(); + protected int frameNumber = 0; + protected long timestamp = 0; + protected int maxBFrames = -1; + protected int trellis = -1; + protected int maxDelay = -1; + + public String getFormat() { + return format; + } + public void setFormat(String format) { + this.format = format; + } + + public String getVideoCodecName() { + return videoCodecName; + } + public void setVideoCodecName(String videoCodecName) { + this.videoCodecName = videoCodecName; + } + + public String getAudioCodecName() { + return audioCodecName; + } + public void setAudioCodecName(String audioCodecName) { + this.audioCodecName = audioCodecName; + } + + public int getImageWidth() { + return imageWidth; + } + public void setImageWidth(int imageWidth) { + this.imageWidth = imageWidth; + } + + public int getImageHeight() { + return imageHeight; + } + public void setImageHeight(int imageHeight) { + this.imageHeight = imageHeight; + } + + public int getAudioChannels() { + return audioChannels; + } + public void setAudioChannels(int audioChannels) { + this.audioChannels = audioChannels; + } + + public int getPixelFormat() { + return pixelFormat; + } + public void setPixelFormat(int pixelFormat) { + this.pixelFormat = pixelFormat; + } + + public int getVideoCodec() { + return videoCodec; + } + public void setVideoCodec(int videoCodec) { + this.videoCodec = videoCodec; + } + + public int getVideoBitrate() { + return videoBitrate; + } + public void setVideoBitrate(int videoBitrate) { + this.videoBitrate = videoBitrate; + } + + public int getImageScalingFlags() { + return imageScalingFlags; + } + public void setImageScalingFlags(int imageScalingFlags) { + this.imageScalingFlags = imageScalingFlags; + } + + public int getGopSize() { + return gopSize; + } + public void setGopSize(int gopSize) { + this.gopSize = gopSize; + } + + public double getAspectRatio() { + return aspectRatio; + } + public void setAspectRatio(double aspectRatio) { + this.aspectRatio = aspectRatio; + } + + public double getFrameRate() { + return frameRate; + } + public void setFrameRate(double frameRate) { + this.frameRate = frameRate; + } + + public double getVideoQuality() { + return videoQuality; + } + public void setVideoQuality(double videoQuality) { + this.videoQuality = videoQuality; + } + + public int getSampleFormat() { + return sampleFormat; + } + public void setSampleFormat(int sampleFormat) { + this.sampleFormat = sampleFormat; + } + + public int getAudioCodec() { + return audioCodec; + } + public void setAudioCodec(int audioCodec) { + this.audioCodec = audioCodec; + } + + public int getAudioBitrate() { + return audioBitrate; + } + public void setAudioBitrate(int audioBitrate) { + this.audioBitrate = audioBitrate; + } + + public int getSampleRate() { + return sampleRate; + } + public void setSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + } + + public double getAudioQuality() { + return audioQuality; + } + public void setAudioQuality(double audioQuality) { + this.audioQuality = audioQuality; + } + + public boolean isInterleaved() { + return interleaved; + } + public void setInterleaved(boolean interleaved) { + this.interleaved = interleaved; + } + + public Charset getCharset() { + return charset; + } + public void setCharset(Charset charset) { + this.charset = charset; + } + + public Map getOptions() { + return options; + } + public void setOptions(Map options) { + this.options = options; + } + + public Map getVideoOptions() { + return videoOptions; + } + public void setVideoOptions(Map options) { + this.videoOptions = options; + } + + public Map getAudioOptions() { + return audioOptions; + } + public void setAudioOptions(Map options) { + this.audioOptions = options; + } + + public Map getMetadata() { + return metadata; + } + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public Map getVideoMetadata() { + return videoMetadata; + } + public void setVideoMetadata(Map metadata) { + this.videoMetadata = metadata; + } + + public Map getAudioMetadata() { + return audioMetadata; + } + public void setAudioMetadata(Map metadata) { + this.audioMetadata = metadata; + } + + public String getOption(String key) { + return options.get(key); + } + public void setOption(String key, String value) { + options.put(key, value); + } + + public String getVideoOption(String key) { + return videoOptions.get(key); + } + public void setVideoOption(String key, String value) { + videoOptions.put(key, value); + } + + public String getAudioOption(String key) { + return audioOptions.get(key); + } + public void setAudioOption(String key, String value) { + audioOptions.put(key, value); + } + + public String getMetadata(String key) { + return metadata.get(key); + } + public void setMetadata(String key, String value) { + metadata.put(key, value); + } + + public String getVideoMetadata(String key) { + return videoMetadata.get(key); + } + public void setVideoMetadata(String key, String value) { + videoMetadata.put(key, value); + } + + public String getAudioMetadata(String key) { + return audioMetadata.get(key); + } + public void setAudioMetadata(String key, String value) { + audioMetadata.put(key, value); + } + + public Map getVideoSideData() { + return videoSideData; + } + public void setVideoSideData(Map videoSideData) { + this.videoSideData = videoSideData; + } + + public Buffer getVideoSideData(String key) { + return videoSideData.get(key); + } + public void setVideoSideData(String key, Buffer value) { + videoSideData.put(key, value); + } + + public Map getAudioSideData() { + return audioSideData; + } + public void setAudioSideData(Map audioSideData) { + this.audioSideData = audioSideData; + } + + public Buffer getAudioSideData(String key) { + return audioSideData.get(key); + } + public void setAudioSideData(String key, Buffer value) { + audioSideData.put(key, value); + } + + public int getFrameNumber() { + return frameNumber; + } + public void setFrameNumber(int frameNumber) { + this.frameNumber = frameNumber; + } + + public long getTimestamp() { + return timestamp; + } + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public int getMaxBFrames() { + return maxBFrames; + } + public void setMaxBFrames(int maxBFrames) { + this.maxBFrames = maxBFrames; + } + + public int getTrellis() { + return trellis; + } + + public void setTrellis(int trellis) { + this.trellis = trellis; + } + + public int getMaxDelay() { + return maxDelay; + } + + public void setMaxDelay(int maxDelay) { + this.maxDelay = maxDelay; + } + + public static class Exception extends IOException { + public Exception(String message) { super(message); } + public Exception(String message, Throwable cause) { super(message, cause); } + } + + public abstract void start() throws Exception; + public abstract void flush() throws Exception; + public abstract void stop() throws Exception; + public abstract void record(Frame frame) throws Exception; + public abstract void release() throws Exception; + + @Override public void close() throws Exception { + stop(); + release(); + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/Java2DFrameConverter.java b/src/main/java/org/bytedeco/javacv/Java2DFrameConverter.java new file mode 100644 index 00000000..365c8f49 --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/Java2DFrameConverter.java @@ -0,0 +1,739 @@ +/* + * Copyright (C) 2015-2019 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +import java.awt.Graphics; +import java.awt.Rectangle; +import java.awt.Transparency; +import java.awt.color.ColorSpace; +import java.awt.image.BufferedImage; +import java.awt.image.ColorModel; +import java.awt.image.ComponentColorModel; +import java.awt.image.ComponentSampleModel; +import java.awt.image.DataBuffer; +import java.awt.image.DataBufferByte; +import java.awt.image.DataBufferDouble; +import java.awt.image.DataBufferFloat; +import java.awt.image.DataBufferInt; +import java.awt.image.DataBufferShort; +import java.awt.image.DataBufferUShort; +import java.awt.image.MultiPixelPackedSampleModel; +import java.awt.image.Raster; +import java.awt.image.SampleModel; +import java.awt.image.SinglePixelPackedSampleModel; +import java.awt.image.WritableRaster; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.DoubleBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; + +/** + * A utility class to copy data between {@link Frame} and {@link BufferedImage}. + * Since {@link BufferedImage} does not support NIO buffers, we cannot share + * allocated memory with {@link Frame}. + * + * @author Samuel Audet + */ +public class Java2DFrameConverter extends FrameConverter { + + @Override public Frame convert(BufferedImage img) { + return getFrame(img); + } + + @Override public BufferedImage convert(Frame frame) { + return getBufferedImage(frame); + } + + /** + * @param source + * @return null if source is null + */ + public static BufferedImage cloneBufferedImage(BufferedImage source) { + if (source == null) { + return null; + } + int type = source.getType(); + if (type == BufferedImage.TYPE_CUSTOM) { + return new BufferedImage( + source.getColorModel(), + source.copyData(null), + source.isAlphaPremultiplied(), + null + ); + } else { + BufferedImage copy = new BufferedImage(source.getWidth(), source.getHeight(), type); + Graphics g = copy.getGraphics(); + g.drawImage(source, 0, 0, null); + g.dispose(); + return copy; + } + } + + public static final byte[] + gamma22 = new byte[256], + gamma22inv = new byte[256]; + static { + for (int i = 0; i < 256; i++) { + gamma22[i] = (byte)Math.round(Math.pow(i/255.0, 2.2)*255.0); + gamma22inv[i] = (byte)Math.round(Math.pow(i/255.0, 1/2.2)*255.0); + } + } + public static int decodeGamma22(int value) { + return gamma22[value & 0xFF] & 0xFF; + } + public static int encodeGamma22(int value) { + return gamma22inv[value & 0xFF] & 0xFF; + } + public static void flipCopyWithGamma(ByteBuffer srcBuf, int srcBufferIndex, int srcStep, + ByteBuffer dstBuf, int dstBufferIndex, int dstStep, + boolean signed, double gamma, boolean flip, int channels) { + assert srcBuf != dstBuf; + int w = Math.min(srcStep, dstStep); + int srcLine = srcBufferIndex, dstLine = dstBufferIndex; + byte[] buffer = new byte[channels]; + while (srcLine < srcBuf.capacity() && dstLine < dstBuf.capacity()) { + if (flip) { + srcBufferIndex = srcBuf.capacity() - srcLine - srcStep; + } else { + srcBufferIndex = srcLine; + } + dstBufferIndex = dstLine; + w = Math.min(Math.min(w, srcBuf.capacity() - srcBufferIndex), dstBuf.capacity() - dstBufferIndex); + if (signed) { + if (channels > 1) { + for (int x = 0; x < w; x+=channels) { + for (int z = 0; z < channels; z++) { + int in = srcBuf.get(srcBufferIndex++); + byte out; + if (gamma == 1.0) { + out = (byte)in; + } else { + out = (byte)Math.round(Math.pow((double)in/Byte.MAX_VALUE, gamma)*Byte.MAX_VALUE); + } + buffer[z] = out; + } + for (int z = channels-1; z >= 0; z--) { + dstBuf.put(dstBufferIndex++, buffer[z]); + } + } + } else { + for (int x = 0; x < w; x++) { + int in = srcBuf.get(srcBufferIndex++); + byte out; + if (gamma == 1.0) { + out = (byte)in; + } else { + out = (byte)Math.round(Math.pow((double)in/Byte.MAX_VALUE, gamma)*Byte.MAX_VALUE); + } + dstBuf.put(dstBufferIndex++, out); + } + } + } else { + if (channels > 1) { + for (int x = 0; x < w; x+=channels) { + for (int z = 0; z < channels; z++) { + byte out; + int in = srcBuf.get(srcBufferIndex++) & 0xFF; + if (gamma == 1.0) { + out = (byte)in; + } else if (gamma == 2.2) { + out = gamma22[in]; + } else if (gamma == 1/2.2) { + out = gamma22inv[in]; + } else { + out = (byte)Math.round(Math.pow((double)in/0xFF, gamma)*0xFF); + } + buffer[z] = out; + } + for (int z = channels-1; z >= 0; z--) { + dstBuf.put(dstBufferIndex++, buffer[z]); + } + } + } else { + for (int x = 0; x < w; x++) { + byte out; + int in = srcBuf.get(srcBufferIndex++) & 0xFF; + if (gamma == 1.0) { + out = (byte)in; + } else if (gamma == 2.2) { + out = gamma22[in]; + } else if (gamma == 1/2.2) { + out = gamma22inv[in]; + } else { + out = (byte)Math.round(Math.pow((double)in/0xFF, gamma)*0xFF); + } + dstBuf.put(dstBufferIndex++, out); + } + } + } + srcLine += srcStep; + dstLine += dstStep; + } + } + public static void flipCopyWithGamma(ShortBuffer srcBuf, int srcBufferIndex, int srcStep, + ShortBuffer dstBuf, int dstBufferIndex, int dstStep, + boolean signed, double gamma, boolean flip, int channels) { + assert srcBuf != dstBuf; + int w = Math.min(srcStep, dstStep); + int srcLine = srcBufferIndex, dstLine = dstBufferIndex; + short[] buffer = new short[channels]; + while (srcLine < srcBuf.capacity() && dstLine < dstBuf.capacity()) { + if (flip) { + srcBufferIndex = srcBuf.capacity() - srcLine - srcStep; + } else { + srcBufferIndex = srcLine; + } + dstBufferIndex = dstLine; + w = Math.min(Math.min(w, srcBuf.capacity() - srcBufferIndex), dstBuf.capacity() - dstBufferIndex); + if (signed) { + if (channels > 1) { + for (int x = 0; x < w; x+=channels) { + for (int z = 0; z < channels; z++) { + int in = srcBuf.get(srcBufferIndex++); + short out; + if (gamma == 1.0) { + out = (short)in; + } else { + out = (short)Math.round(Math.pow((double)in/Short.MAX_VALUE, gamma)*Short.MAX_VALUE); + } + buffer[z] = out; + } + for (int z = channels-1; z >= 0; z--) { + dstBuf.put(dstBufferIndex++, buffer[z]); + } + } + } else { + for (int x = 0; x < w; x++) { + int in = srcBuf.get(srcBufferIndex++); + short out; + if (gamma == 1.0) { + out = (short)in; + } else { + out = (short)Math.round(Math.pow((double)in/Short.MAX_VALUE, gamma)*Short.MAX_VALUE); + } + dstBuf.put(dstBufferIndex++, out); + } + } + } else { + if (channels > 1) { + for (int x = 0; x < w; x+=channels) { + for (int z = 0; z < channels; z++) { + int in = srcBuf.get(srcBufferIndex++); + short out; + if (gamma == 1.0) { + out = (short)in; + } else { + out = (short)Math.round(Math.pow((double)in/0xFFFF, gamma)*0xFFFF); + } + buffer[z] = out; + } + for (int z = channels-1; z >= 0; z--) { + dstBuf.put(dstBufferIndex++, buffer[z]); + } + } + } else { + for (int x = 0; x < w; x++) { + int in = srcBuf.get(srcBufferIndex++) & 0xFFFF; + short out; + if (gamma == 1.0) { + out = (short)in; + } else { + out = (short)Math.round(Math.pow((double)in/0xFFFF, gamma)*0xFFFF); + } + dstBuf.put(dstBufferIndex++, out); + } + } + } + srcLine += srcStep; + dstLine += dstStep; + } + } + public static void flipCopyWithGamma(IntBuffer srcBuf, int srcBufferIndex, int srcStep, + IntBuffer dstBuf, int dstBufferIndex, int dstStep, + double gamma, boolean flip, int channels) { + assert srcBuf != dstBuf; + int w = Math.min(srcStep, dstStep); + int srcLine = srcBufferIndex, dstLine = dstBufferIndex; + int[] buffer = new int[channels]; + while (srcLine < srcBuf.capacity() && dstLine < dstBuf.capacity()) { + if (flip) { + srcBufferIndex = srcBuf.capacity() - srcLine - srcStep; + } else { + srcBufferIndex = srcLine; + } + dstBufferIndex = dstLine; + w = Math.min(Math.min(w, srcBuf.capacity() - srcBufferIndex), dstBuf.capacity() - dstBufferIndex); + if (channels > 1) { + for (int x = 0; x < w; x+=channels) { + for (int z = 0; z < channels; z++) { + int in = srcBuf.get(srcBufferIndex++); + int out; + if (gamma == 1.0) { + out = (int)in; + } else { + out = (int)Math.round(Math.pow((double)in/Integer.MAX_VALUE, gamma)*Integer.MAX_VALUE); + } + buffer[z] = out; + } + for (int z = channels-1; z >= 0; z--) { + dstBuf.put(dstBufferIndex++, buffer[z]); + } + } + } else { + for (int x = 0; x < w; x++) { + int in = srcBuf.get(srcBufferIndex++); + int out; + if (gamma == 1.0) { + out = in; + } else { + out = (int)Math.round(Math.pow((double)in/Integer.MAX_VALUE, gamma)*Integer.MAX_VALUE); + } + dstBuf.put(dstBufferIndex++, out); + } + } + srcLine += srcStep; + dstLine += dstStep; + } + } + public static void flipCopyWithGamma(FloatBuffer srcBuf, int srcBufferIndex, int srcStep, + FloatBuffer dstBuf, int dstBufferIndex, int dstStep, + double gamma, boolean flip, int channels) { + assert srcBuf != dstBuf; + int w = Math.min(srcStep, dstStep); + int srcLine = srcBufferIndex, dstLine = dstBufferIndex; + float[] buffer = new float[channels]; + while (srcLine < srcBuf.capacity() && dstLine < dstBuf.capacity()) { + if (flip) { + srcBufferIndex = srcBuf.capacity() - srcLine - srcStep; + } else { + srcBufferIndex = srcLine; + } + dstBufferIndex = dstLine; + w = Math.min(Math.min(w, srcBuf.capacity() - srcBufferIndex), dstBuf.capacity() - dstBufferIndex); + if (channels > 1) { + for (int x = 0; x < w; x+=channels) { + for (int z = 0; z < channels; z++) { + float in = srcBuf.get(srcBufferIndex++); + float out; + if (gamma == 1.0) { + out = in; + } else { + out = (float)Math.pow(in, gamma); + } + buffer[z] = out; + } + for (int z = channels-1; z >= 0; z--) { + dstBuf.put(dstBufferIndex++, buffer[z]); + } + } + } else { + for (int x = 0; x < w; x++) { + float in = srcBuf.get(srcBufferIndex++); + float out; + if (gamma == 1.0) { + out = in; + } else { + out = (float)Math.pow(in, gamma); + } + dstBuf.put(dstBufferIndex++, out); + } + } + srcLine += srcStep; + dstLine += dstStep; + } + } + public static void flipCopyWithGamma(DoubleBuffer srcBuf, int srcBufferIndex, int srcStep, + DoubleBuffer dstBuf, int dstBufferIndex, int dstStep, + double gamma, boolean flip, int channels) { + assert srcBuf != dstBuf; + int w = Math.min(srcStep, dstStep); + int srcLine = srcBufferIndex, dstLine = dstBufferIndex; + double[] buffer = new double[channels]; + while (srcLine < srcBuf.capacity() && dstLine < dstBuf.capacity()) { + if (flip) { + srcBufferIndex = srcBuf.capacity() - srcLine - srcStep; + } else { + srcBufferIndex = srcLine; + } + dstBufferIndex = dstLine; + w = Math.min(Math.min(w, srcBuf.capacity() - srcBufferIndex), dstBuf.capacity() - dstBufferIndex); + if (channels > 1) { + for (int x = 0; x < w; x+=channels) { + for (int z = 0; z < channels; z++) { + double in = srcBuf.get(srcBufferIndex++); + double out; + if (gamma == 1.0) { + out = in; + } else { + out = Math.pow(in, gamma); + } + buffer[z] = out; + } + for (int z = channels-1; z >= 0; z--) { + dstBuf.put(dstBufferIndex++, buffer[z]); + } + } + } else { + for (int x = 0; x < w; x++) { + double in = srcBuf.get(srcBufferIndex++); + double out; + if (gamma == 1.0) { + out = in; + } else { + out = Math.pow(in, gamma); + } + dstBuf.put(dstBufferIndex++, out); + } + } + srcLine += srcStep; + dstLine += dstStep; + } + } + + public static void applyGamma(Frame frame, double gamma) { + applyGamma(frame.image[0], frame.imageDepth, frame.imageStride, gamma); + } + public static void applyGamma(Buffer buffer, int depth, int stride, double gamma) { + if (gamma == 1.0) { + return; + } + switch (depth) { + case Frame.DEPTH_UBYTE: + flipCopyWithGamma(((ByteBuffer)buffer).asReadOnlyBuffer(), 0, stride, (ByteBuffer)buffer, 0, stride, false, gamma, false, 0); + break; + case Frame.DEPTH_BYTE: + flipCopyWithGamma(((ByteBuffer)buffer).asReadOnlyBuffer(), 0, stride, (ByteBuffer)buffer, 0, stride, true, gamma, false, 0); + break; + case Frame.DEPTH_USHORT: + flipCopyWithGamma(((ShortBuffer)buffer).asReadOnlyBuffer(), 0, stride, (ShortBuffer)buffer, 0, stride, false, gamma, false, 0); + break; + case Frame.DEPTH_SHORT: + flipCopyWithGamma(((ShortBuffer)buffer).asReadOnlyBuffer(), 0, stride, (ShortBuffer)buffer, 0, stride, true, gamma, false, 0); + break; + case Frame.DEPTH_INT: + flipCopyWithGamma(((IntBuffer)buffer).asReadOnlyBuffer(), 0, stride, (IntBuffer)buffer, 0, stride, gamma, false, 0); + break; + case Frame.DEPTH_FLOAT: + flipCopyWithGamma(((FloatBuffer)buffer).asReadOnlyBuffer(), 0, stride, (FloatBuffer)buffer, 0, stride, gamma, false, 0); + break; + case Frame.DEPTH_DOUBLE: + flipCopyWithGamma(((DoubleBuffer)buffer).asReadOnlyBuffer(), 0, stride, (DoubleBuffer)buffer, 0, stride, gamma, false, 0); + break; + default: + assert false; + } + } + + public static void copy(Frame frame, BufferedImage bufferedImage) { + copy(frame, bufferedImage, 1.0); + } + public static void copy(Frame frame, BufferedImage bufferedImage, double gamma) { + copy(frame, bufferedImage, gamma, false, null); + } + public static void copy(Frame frame, BufferedImage bufferedImage, double gamma, boolean flipChannels, Rectangle roi) { + Buffer in = frame.image[0]; + int bufferIndex = roi == null ? 0 : roi.y*frame.imageStride + roi.x*frame.imageChannels; + SampleModel sm = bufferedImage.getSampleModel(); + Raster r = bufferedImage.getRaster(); + DataBuffer out = r.getDataBuffer(); + int x = -r.getSampleModelTranslateX(); + int y = -r.getSampleModelTranslateY(); + int step = sm.getWidth()*sm.getNumBands(); + int channels = sm.getNumBands(); + if (sm instanceof ComponentSampleModel) { + step = ((ComponentSampleModel)sm).getScanlineStride(); + channels = ((ComponentSampleModel)sm).getPixelStride(); + } else if (sm instanceof SinglePixelPackedSampleModel) { + step = ((SinglePixelPackedSampleModel)sm).getScanlineStride(); + channels = 1; + } else if (sm instanceof MultiPixelPackedSampleModel) { + step = ((MultiPixelPackedSampleModel)sm).getScanlineStride(); + channels = ((MultiPixelPackedSampleModel)sm).getPixelBitStride()/8; // ?? + } + int start = y*step + x*channels; + + if (out instanceof DataBufferByte) { + byte[] a = ((DataBufferByte)out).getData(); + flipCopyWithGamma((ByteBuffer)in, bufferIndex, frame.imageStride, ByteBuffer.wrap(a), start, step, false, gamma, false, flipChannels ? channels : 0); + } else if (out instanceof DataBufferDouble) { + double[] a = ((DataBufferDouble)out).getData(); + flipCopyWithGamma((DoubleBuffer)in, bufferIndex, frame.imageStride, DoubleBuffer.wrap(a), start, step, gamma, false, flipChannels ? channels : 0); + } else if (out instanceof DataBufferFloat) { + float[] a = ((DataBufferFloat)out).getData(); + flipCopyWithGamma((FloatBuffer)in, bufferIndex, frame.imageStride, FloatBuffer.wrap(a), start, step, gamma, false, flipChannels ? channels : 0); + } else if (out instanceof DataBufferInt) { + int[] a = ((DataBufferInt)out).getData(); + int stride = frame.imageStride; + if (in instanceof ByteBuffer) { + in = ((ByteBuffer)in).order(flipChannels ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN).asIntBuffer(); + stride /= 4; + } + flipCopyWithGamma((IntBuffer)in, bufferIndex, stride, IntBuffer.wrap(a), start, step, gamma, false, flipChannels ? channels : 0); + } else if (out instanceof DataBufferShort) { + short[] a = ((DataBufferShort)out).getData(); + flipCopyWithGamma((ShortBuffer)in, bufferIndex, frame.imageStride, ShortBuffer.wrap(a), start, step, true, gamma, false, flipChannels ? channels : 0); + } else if (out instanceof DataBufferUShort) { + short[] a = ((DataBufferUShort)out).getData(); + flipCopyWithGamma((ShortBuffer)in, bufferIndex, frame.imageStride, ShortBuffer.wrap(a), start, step, false, gamma, false, flipChannels ? channels : 0); + } else { + assert false; + } + } + + public static void copy(BufferedImage image, Frame frame) { + copy(image, frame, 1.0); + } + public static void copy(BufferedImage image, Frame frame, double gamma) { + copy(image, frame, gamma, false, null); + } + public static void copy(BufferedImage image, Frame frame, double gamma, boolean flipChannels, Rectangle roi) { + Buffer out = frame.image[0]; + int bufferIndex = roi == null ? 0 : roi.y*frame.imageStride + roi.x*frame.imageChannels; + SampleModel sm = image.getSampleModel(); + Raster r = image.getRaster(); + DataBuffer in = r.getDataBuffer(); + int x = -r.getSampleModelTranslateX(); + int y = -r.getSampleModelTranslateY(); + int step = sm.getWidth()*sm.getNumBands(); + int channels = sm.getNumBands(); + if (sm instanceof ComponentSampleModel) { + step = ((ComponentSampleModel)sm).getScanlineStride(); + channels = ((ComponentSampleModel)sm).getPixelStride(); + } else if (sm instanceof SinglePixelPackedSampleModel) { + step = ((SinglePixelPackedSampleModel)sm).getScanlineStride(); + channels = 1; + } else if (sm instanceof MultiPixelPackedSampleModel) { + step = ((MultiPixelPackedSampleModel)sm).getScanlineStride(); + channels = ((MultiPixelPackedSampleModel)sm).getPixelBitStride()/8; // ?? + } + int start = y*step + x*channels; + + if (in instanceof DataBufferByte) { + byte[] a = ((DataBufferByte)in).getData(); + flipCopyWithGamma(ByteBuffer.wrap(a), start, step, (ByteBuffer)out, bufferIndex, frame.imageStride, false, gamma, false, flipChannels ? channels : 0); + } else if (in instanceof DataBufferDouble) { + double[] a = ((DataBufferDouble)in).getData(); + flipCopyWithGamma(DoubleBuffer.wrap(a), start, step, (DoubleBuffer)out, bufferIndex, frame.imageStride, gamma, false, flipChannels ? channels : 0); + } else if (in instanceof DataBufferFloat) { + float[] a = ((DataBufferFloat)in).getData(); + flipCopyWithGamma(FloatBuffer.wrap(a), start, step, (FloatBuffer)out, bufferIndex, frame.imageStride, gamma, false, flipChannels ? channels : 0); + } else if (in instanceof DataBufferInt) { + int[] a = ((DataBufferInt)in).getData(); + int stride = frame.imageStride; + if (out instanceof ByteBuffer) { + out = ((ByteBuffer)out).order(flipChannels ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN).asIntBuffer(); + stride /= 4; + } + flipCopyWithGamma(IntBuffer.wrap(a), start, step, (IntBuffer)out, bufferIndex, stride, gamma, false, flipChannels ? channels : 0); + } else if (in instanceof DataBufferShort) { + short[] a = ((DataBufferShort)in).getData(); + flipCopyWithGamma(ShortBuffer.wrap(a), start, step, (ShortBuffer)out, bufferIndex, frame.imageStride, true, gamma, false, flipChannels ? channels : 0); + } else if (in instanceof DataBufferUShort) { + short[] a = ((DataBufferUShort)in).getData(); + flipCopyWithGamma(ShortBuffer.wrap(a), start, step, (ShortBuffer)out, bufferIndex, frame.imageStride, false, gamma, false, flipChannels ? channels : 0); + } else { + assert false; + } + } + + protected BufferedImage bufferedImage = null; + public static int getBufferedImageType(Frame frame) { + // precanned BufferedImage types are confusing... in practice though, + // they all use the sRGB color model when blitting: + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=5051418 + // and we should use them because they are *A LOT* faster with Java 2D. + // workaround: do gamma correction ourselves ("gamma" parameter) + // since we'll never use getRGB() and setRGB(), right? + int type = BufferedImage.TYPE_CUSTOM; + if (frame.imageChannels == 1) { + if (frame.imageDepth == Frame.DEPTH_UBYTE || frame.imageDepth == Frame.DEPTH_BYTE) { + type = BufferedImage.TYPE_BYTE_GRAY; + } else if (frame.imageDepth == Frame.DEPTH_USHORT) { + type = BufferedImage.TYPE_USHORT_GRAY; + } + } else if (frame.imageChannels == 3) { + if (frame.imageDepth == Frame.DEPTH_UBYTE || frame.imageDepth == Frame.DEPTH_BYTE) { + type = BufferedImage.TYPE_3BYTE_BGR; + } + } else if (frame.imageChannels == 4) { + // The channels end up reversed of what we need for OpenCL. + // We work around this in copyTo() and copyFrom() by + // inversing the channels to let us use RGBA in our IplImage. + if (frame.imageDepth == Frame.DEPTH_UBYTE || frame.imageDepth == Frame.DEPTH_BYTE) { + type = BufferedImage.TYPE_4BYTE_ABGR; + } + } + return type; + } + public BufferedImage getBufferedImage(Frame frame) { + return getBufferedImage(frame, 1.0); + } + public BufferedImage getBufferedImage(Frame frame, double gamma) { + return getBufferedImage(frame, gamma, false, null); + } + public BufferedImage getBufferedImage(Frame frame, double gamma, boolean flipChannels, ColorSpace cs) { + if (frame == null || frame.image == null) { + return null; + } + int type = getBufferedImageType(frame); + + if (bufferedImage == null || bufferedImage.getWidth() != frame.imageWidth + || bufferedImage.getHeight() != frame.imageHeight || bufferedImage.getType() != type) { + bufferedImage = type == BufferedImage.TYPE_CUSTOM || cs != null ? null + : new BufferedImage(frame.imageWidth, frame.imageHeight, type); + } + + if (bufferedImage == null) { + boolean alpha = false; + int[] offsets = null; + if (frame.imageChannels == 1) { + alpha = false; + if (cs == null) { + cs = ColorSpace.getInstance(ColorSpace.CS_GRAY); + } + offsets = new int[] {0}; + } else if (frame.imageChannels == 3) { + alpha = false; + if (cs == null) { + cs = ColorSpace.getInstance(ColorSpace.CS_LINEAR_RGB); + } + // raster in "BGR" order like OpenCV.. + offsets = new int[] {2, 1, 0}; + } else if (frame.imageChannels == 4) { + alpha = true; + if (cs == null) { + cs = ColorSpace.getInstance(ColorSpace.CS_LINEAR_RGB); + } + // raster in "RGBA" order for OpenCL.. alpha needs to be last + offsets = new int[] {0, 1, 2, 3}; + } else { + assert false; + } + + ColorModel cm = null; + WritableRaster wr = null; + if (frame.imageDepth == Frame.DEPTH_UBYTE || frame.imageDepth == Frame.DEPTH_BYTE) { + cm = new ComponentColorModel(cs, alpha, + false, Transparency.OPAQUE, DataBuffer.TYPE_BYTE); + wr = Raster.createWritableRaster(new ComponentSampleModel( + DataBuffer.TYPE_BYTE, frame.imageWidth, frame.imageHeight, frame.imageChannels, frame.imageStride, + offsets), null); + } else if (frame.imageDepth == Frame.DEPTH_USHORT) { + cm = new ComponentColorModel(cs, alpha, + false, Transparency.OPAQUE, DataBuffer.TYPE_USHORT); + wr = Raster.createWritableRaster(new ComponentSampleModel( + DataBuffer.TYPE_USHORT, frame.imageWidth, frame.imageHeight, frame.imageChannels, frame.imageStride, + offsets), null); + } else if (frame.imageDepth == Frame.DEPTH_SHORT) { + cm = new ComponentColorModel(cs, alpha, + false, Transparency.OPAQUE, DataBuffer.TYPE_SHORT); + wr = Raster.createWritableRaster(new ComponentSampleModel( + DataBuffer.TYPE_SHORT, frame.imageWidth, frame.imageHeight, frame.imageChannels, frame.imageStride, + offsets), null); + } else if (frame.imageDepth == Frame.DEPTH_INT) { + cm = new ComponentColorModel(cs, alpha, + false, Transparency.OPAQUE, DataBuffer.TYPE_INT); + wr = Raster.createWritableRaster(new ComponentSampleModel( + DataBuffer.TYPE_INT, frame.imageWidth, frame.imageHeight, frame.imageChannels, frame.imageStride, + offsets), null); + } else if (frame.imageDepth == Frame.DEPTH_FLOAT) { + cm = new ComponentColorModel(cs, alpha, + false, Transparency.OPAQUE, DataBuffer.TYPE_FLOAT); + wr = Raster.createWritableRaster(new ComponentSampleModel( + DataBuffer.TYPE_FLOAT, frame.imageWidth, frame.imageHeight, frame.imageChannels, frame.imageStride, + offsets), null); + } else if (frame.imageDepth == Frame.DEPTH_DOUBLE) { + cm = new ComponentColorModel(cs, alpha, + false, Transparency.OPAQUE, DataBuffer.TYPE_DOUBLE); + wr = Raster.createWritableRaster(new ComponentSampleModel( + DataBuffer.TYPE_DOUBLE, frame.imageWidth, frame.imageHeight, frame.imageChannels, frame.imageStride, + offsets), null); + } else { + assert false; + } + + bufferedImage = new BufferedImage(cm, wr, false, null); + } + + if (bufferedImage != null) { + copy(frame, bufferedImage, gamma, flipChannels, null); + } + + return bufferedImage; + } + + /** + * Returns a Frame based on a BufferedImage. + */ + public Frame getFrame(BufferedImage image) { + return getFrame(image, 1.0); + } + /** + * Returns a Frame based on a BufferedImage, and given gamma. + */ + public Frame getFrame(BufferedImage image, double gamma) { + return getFrame(image, gamma, false); + } + /** + * Returns a Frame based on a BufferedImage, given gamma, and inverted channels flag. + */ + public Frame getFrame(BufferedImage image, double gamma, boolean flipChannels) { + if (image == null) { + return null; + } + SampleModel sm = image.getSampleModel(); + int depth = 0, numChannels = sm.getNumBands(); + switch (image.getType()) { + case BufferedImage.TYPE_INT_RGB: + case BufferedImage.TYPE_INT_ARGB: + case BufferedImage.TYPE_INT_ARGB_PRE: + case BufferedImage.TYPE_INT_BGR: + depth = Frame.DEPTH_UBYTE; + numChannels = 4; + break; + } + if (depth == 0 || numChannels == 0) { + switch (sm.getDataType()) { + case DataBuffer.TYPE_BYTE: depth = Frame.DEPTH_UBYTE; break; + case DataBuffer.TYPE_USHORT: depth = Frame.DEPTH_USHORT; break; + case DataBuffer.TYPE_SHORT: depth = Frame.DEPTH_SHORT; break; + case DataBuffer.TYPE_INT: depth = Frame.DEPTH_INT; break; + case DataBuffer.TYPE_FLOAT: depth = Frame.DEPTH_FLOAT; break; + case DataBuffer.TYPE_DOUBLE: depth = Frame.DEPTH_DOUBLE; break; + default: assert false; + } + } + if (frame == null || frame.imageWidth != image.getWidth() || frame.imageHeight != image.getHeight() + || frame.imageDepth != depth || frame.imageChannels != numChannels) { + if (frame != null) { + frame.close(); + } + frame = new Frame(image.getWidth(), image.getHeight(), depth, numChannels); + } + copy(image, frame, gamma, flipChannels, null); + return frame; + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/OpenCVFrameConverter.java b/src/main/java/org/bytedeco/javacv/OpenCVFrameConverter.java new file mode 100644 index 00000000..1b8bd4ea --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/OpenCVFrameConverter.java @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2015-2021 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.Pointer; + +import org.bytedeco.opencv.opencv_core.*; +import static org.bytedeco.opencv.global.opencv_core.*; + +/** + * A utility class to map data between {@link Frame} and {@link IplImage} or {@link Mat}. + * Since this is an abstract class, one must choose between two concrete classes: + * {@link ToIplImage} or {@link ToMat}. {@link ToOrgOpenCvCoreMat} is also available to + * do the same with {@link org.opencv.core.Mat} from the official Java API of OpenCV. + * + * @author Samuel Audet + */ +public abstract class OpenCVFrameConverter extends FrameConverter { + static { Loader.load(org.bytedeco.opencv.global.opencv_core.class); } + + IplImage img; + Mat mat; + org.opencv.core.Mat orgOpenCvCoreMat; + + public static class ToIplImage extends OpenCVFrameConverter { + @Override public Frame convert(IplImage img) { return super.convert(img); } + @Override public IplImage convert(Frame frame) { return convertToIplImage(frame); } + } + + public static class ToMat extends OpenCVFrameConverter { + @Override public Frame convert(Mat mat) { return super.convert(mat); } + @Override public Mat convert(Frame frame) { return convertToMat(frame); } + } + + public static class ToOrgOpenCvCoreMat extends OpenCVFrameConverter { + @Override public Frame convert(org.opencv.core.Mat mat) { return super.convert(mat); } + @Override public org.opencv.core.Mat convert(Frame frame) { return convertToOrgOpenCvCoreMat(frame); } + } + + public static int getFrameDepth(int depth) { + switch (depth) { + case IPL_DEPTH_8U: case CV_8U: return Frame.DEPTH_UBYTE; + case IPL_DEPTH_8S: case CV_8S: return Frame.DEPTH_BYTE; + case IPL_DEPTH_16U: case CV_16U: return Frame.DEPTH_USHORT; + case IPL_DEPTH_16S: case CV_16S: return Frame.DEPTH_SHORT; + case IPL_DEPTH_32F: case CV_32F: return Frame.DEPTH_FLOAT; + case IPL_DEPTH_32S: case CV_32S: return Frame.DEPTH_INT; + case IPL_DEPTH_64F: case CV_64F: return Frame.DEPTH_DOUBLE; + default: return -1; + } + } + + public static int getIplImageDepth(int depth) { + switch (depth) { + case Frame.DEPTH_UBYTE: return IPL_DEPTH_8U; + case Frame.DEPTH_BYTE: return IPL_DEPTH_8S; + case Frame.DEPTH_USHORT: return IPL_DEPTH_16U; + case Frame.DEPTH_SHORT: return IPL_DEPTH_16S; + case Frame.DEPTH_FLOAT: return IPL_DEPTH_32F; + case Frame.DEPTH_INT: return IPL_DEPTH_32S; + case Frame.DEPTH_DOUBLE: return IPL_DEPTH_64F; + default: return -1; + } + } + static boolean isEqual(Frame frame, IplImage img) { + return img != null && frame != null && frame.image != null && frame.image.length > 0 + && frame.imageWidth == img.width() && frame.imageHeight == img.height() + && frame.imageChannels == img.nChannels() && getIplImageDepth(frame.imageDepth) == img.depth() + && new Pointer(frame.image[0].position(0)).address() == img.imageData().address() + && frame.imageStride * Math.abs(frame.imageDepth) / 8 == img.widthStep(); + } + public IplImage convertToIplImage(Frame frame) { + if (frame == null || frame.image == null) { + return null; + } else if (frame.opaque instanceof IplImage) { + return (IplImage)frame.opaque; + } else if (!isEqual(frame, img)) { + int depth = getIplImageDepth(frame.imageDepth); + if (img != null) { + img.releaseReference(); + } + img = depth < 0 ? null : (IplImage)IplImage.create(frame.imageWidth, frame.imageHeight, depth, frame.imageChannels, new Pointer(frame.image[0].position(0))) + .widthStep(frame.imageStride * Math.abs(frame.imageDepth) / 8) + .imageSize(frame.image[0].capacity() * Math.abs(frame.imageDepth) / 8).retainReference(); + } + return img; + } + public Frame convert(IplImage img) { + if (img == null) { + return null; + } else if (!isEqual(frame, img)) { + frame = new Frame(); + frame.imageWidth = img.width(); + frame.imageHeight = img.height(); + frame.imageDepth = getFrameDepth(img.depth()); + frame.imageChannels = img.nChannels(); + frame.imageStride = img.widthStep() * 8 / Math.abs(frame.imageDepth); + frame.image = new Buffer[] { img.createBuffer() }; + } + frame.opaque = img; + return frame; + } + + public static int getMatDepth(int depth) { + switch (depth) { + case Frame.DEPTH_UBYTE: return CV_8U; + case Frame.DEPTH_BYTE: return CV_8S; + case Frame.DEPTH_USHORT: return CV_16U; + case Frame.DEPTH_SHORT: return CV_16S; + case Frame.DEPTH_FLOAT: return CV_32F; + case Frame.DEPTH_INT: return CV_32S; + case Frame.DEPTH_DOUBLE: return CV_64F; + default: return -1; + } + } + static boolean isEqual(Frame frame, Mat mat) { + return mat != null && frame != null && frame.image != null && frame.image.length > 0 + && frame.imageWidth == mat.cols() && frame.imageHeight == mat.rows() + && frame.imageChannels == mat.channels() && getMatDepth(frame.imageDepth) == mat.depth() + && new Pointer(frame.image[0].position(0)).address() == mat.data().address() + && frame.imageStride * Math.abs(frame.imageDepth) / 8 == (int)mat.step(); + } + public Mat convertToMat(Frame frame) { + if (frame == null || frame.image == null) { + return null; + } else if (frame.opaque instanceof Mat) { + return (Mat)frame.opaque; + } else if (!isEqual(frame, mat)) { + int depth = getMatDepth(frame.imageDepth); + if (mat != null) { + mat.releaseReference(); + } + mat = depth < 0 ? null : (Mat)new Mat(frame.imageHeight, frame.imageWidth, CV_MAKETYPE(depth, frame.imageChannels), + new Pointer(frame.image[0].position(0)), frame.imageStride * Math.abs(frame.imageDepth) / 8).retainReference(); + } + return mat; + } + public Frame convert(Mat mat) { + if (mat == null) { + return null; + } else if (!isEqual(frame, mat)) { + frame = new Frame(); + frame.imageWidth = mat.cols(); + frame.imageHeight = mat.rows(); + frame.imageDepth = getFrameDepth(mat.depth()); + frame.imageChannels = mat.channels(); + frame.imageStride = (int)mat.step() * 8 / Math.abs(frame.imageDepth); + frame.image = new Buffer[] { mat.createBuffer() }; + } + frame.opaque = mat; + return frame; + } + + static boolean isEqual(Frame frame, org.opencv.core.Mat mat) { + return mat != null && frame != null && frame.image != null && frame.image.length > 0 + && frame.imageWidth == mat.cols() && frame.imageHeight == mat.rows() + && frame.imageChannels == mat.channels() && getMatDepth(frame.imageDepth) == mat.depth() + && new Pointer(frame.image[0].position(0)).address() == mat.dataAddr(); + } + public org.opencv.core.Mat convertToOrgOpenCvCoreMat(Frame frame) { + if (frame == null || frame.image == null) { + return null; + } else if (frame.opaque instanceof org.opencv.core.Mat) { + return (org.opencv.core.Mat)frame.opaque; + } else if (!isEqual(frame, mat)) { + int depth = getMatDepth(frame.imageDepth); + orgOpenCvCoreMat = depth < 0 ? null : new org.opencv.core.Mat(frame.imageHeight, frame.imageWidth, + CV_MAKETYPE(depth, frame.imageChannels), new BytePointer(new Pointer(frame.image[0].position(0))) + .capacity(frame.image[0].capacity() * Math.abs(frame.imageDepth) / 8).asByteBuffer(), + frame.imageStride * Math.abs(frame.imageDepth) / 8); + } + return orgOpenCvCoreMat; + } + public Frame convert(final org.opencv.core.Mat mat) { + if (mat == null) { + return null; + } else if (!isEqual(frame, mat)) { + frame = new Frame(); + frame.imageWidth = mat.cols(); + frame.imageHeight = mat.rows(); + frame.imageDepth = getFrameDepth(mat.depth()); + frame.imageChannels = mat.channels(); + frame.imageStride = (int)mat.step1(); + ByteBuffer byteBuffer = new BytePointer() { { address = mat.dataAddr(); } }.capacity(mat.rows() * mat.step1() * mat.elemSize1()).asByteBuffer(); + switch (mat.depth()) { + case CV_8U: + case CV_8S: + frame.image = new Buffer[] { byteBuffer }; + break; + case CV_16U: + case CV_16S: + frame.image = new Buffer[] { byteBuffer.asShortBuffer() }; + break; + case CV_32F: + frame.image = new Buffer[] { byteBuffer.asFloatBuffer() }; + break; + case CV_32S: + frame.image = new Buffer[] { byteBuffer.asIntBuffer() }; + break; + case CV_64F: + frame.image = new Buffer[] { byteBuffer.asDoubleBuffer() }; + break; + default: + frame.image = null; + break; + } + } + frame.opaque = mat; + return frame; + } + + @Override public void close() { + super.close(); + if (img != null) { + img.releaseReference(); + img = null; + } + if (mat != null) { + mat.releaseReference(); + mat = null; + } + if (orgOpenCvCoreMat != null) { + orgOpenCvCoreMat.release(); + orgOpenCvCoreMat = null; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/OpenCVFrameGrabber.java b/src/main/java/org/bytedeco/javacv/OpenCVFrameGrabber.java new file mode 100644 index 00000000..6a9c2728 --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/OpenCVFrameGrabber.java @@ -0,0 +1,318 @@ +/* + * Copyright (C) 2009-2019 Samuel Audet + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +import java.io.File; +import java.util.Map; +import java.util.Map.Entry; +import org.bytedeco.javacpp.Loader; + +import org.bytedeco.opencv.opencv_core.*; +import org.bytedeco.opencv.opencv_imgproc.*; +import org.bytedeco.opencv.opencv_videoio.*; +import static org.bytedeco.opencv.global.opencv_core.*; +import static org.bytedeco.opencv.global.opencv_imgproc.*; +import static org.bytedeco.opencv.global.opencv_videoio.*; + +/** + * + * @author Samuel Audet + * @author Lloyd (github.com/lloydmeta) + */ +public class OpenCVFrameGrabber extends FrameGrabber { + public static String[] getDeviceDescriptions() throws Exception { + tryLoad(); + throw new UnsupportedOperationException("Device enumeration not support by OpenCV."); + } + + public static OpenCVFrameGrabber createDefault(File deviceFile) throws Exception { return new OpenCVFrameGrabber(deviceFile); } + public static OpenCVFrameGrabber createDefault(String devicePath) throws Exception { return new OpenCVFrameGrabber(devicePath); } + public static OpenCVFrameGrabber createDefault(int deviceNumber) throws Exception { return new OpenCVFrameGrabber(deviceNumber); } + + private static Exception loadingException = null; + public static void tryLoad() throws Exception { + if (loadingException != null) { + throw loadingException; + } else { + try { + Loader.load(org.bytedeco.opencv.global.opencv_highgui.class); + } catch (Throwable t) { + throw loadingException = new Exception("Failed to load " + OpenCVFrameGrabber.class, t); + } + } + } + + public OpenCVFrameGrabber(int deviceNumber) { + this.deviceNumber = deviceNumber; + } + public OpenCVFrameGrabber(File file) { + this(file.getAbsolutePath()); + } + public OpenCVFrameGrabber(File file, int apiPreference) { + this(file.getAbsolutePath(), apiPreference); + } + public OpenCVFrameGrabber(String filename) { + this.filename = filename; + } + public OpenCVFrameGrabber(String filename, int apiPreference) { + this.filename = filename; + this.apiPreference = apiPreference; + } + + public void release() throws Exception { + stop(); + } + @Override protected void finalize() throws Throwable { + super.finalize(); + release(); + } + + private int deviceNumber = 0; + private String filename = null; + private int apiPreference = 0; + private VideoCapture capture = null; + private Mat returnMatrix = null; + private final OpenCVFrameConverter converter = new OpenCVFrameConverter.ToMat(); + private final Mat mat = new Mat(); + + @Override public double getGamma() { + // default to a gamma of 2.2 for cheap Webcams, DV cameras, etc. + if (gamma == 0.0) { + return 2.2; + } else { + return gamma; + } + } + + @Override public String getFormat() { + if (capture == null) { + return super.getFormat(); + } else { + int fourcc = (int)capture.get(CAP_PROP_FOURCC); + return "" + (char)( fourcc & 0xFF) + + (char)((fourcc >> 8) & 0xFF) + + (char)((fourcc >> 16) & 0xFF) + + (char)((fourcc >> 24) & 0xFF); + } + } + + @Override public int getImageWidth() { + if (returnMatrix != null) { + return returnMatrix.cols(); + } else { + return capture == null ? super.getImageWidth() : (int)capture.get(CAP_PROP_FRAME_WIDTH); + } + } + + @Override public int getImageHeight() { + if (returnMatrix != null) { + return returnMatrix.rows(); + } else { + return capture == null ? super.getImageHeight() : (int)capture.get(CAP_PROP_FRAME_HEIGHT); + } + } + + @Override public int getPixelFormat() { + return capture == null ? super.getPixelFormat() : (int)capture.get(CAP_PROP_CONVERT_RGB); + } + + @Override public double getFrameRate() { + return capture == null ? super.getFrameRate() : (int)capture.get(CAP_PROP_FPS); + } + + @Override public void setImageMode(ImageMode imageMode) { + if (imageMode != this.imageMode) { + returnMatrix = null; + } + super.setImageMode(imageMode); + } + + @Override public int getFrameNumber() { + return capture == null ? super.getFrameNumber() : + (int)capture.get(CAP_PROP_POS_FRAMES); + } + @Override public void setFrameNumber(int frameNumber) throws Exception { + if (capture == null) { + super.setFrameNumber(frameNumber); + } else { + if (!capture.set(CAP_PROP_POS_FRAMES, frameNumber)) { + throw new Exception("set() Error: Could not set CAP_PROP_POS_FRAMES to " + frameNumber + "."); + } + } + } + + @Override public long getTimestamp() { + return capture == null ? super.getTimestamp() : + Math.round(capture.get(CAP_PROP_POS_MSEC)*1000); + } + @Override public void setTimestamp(long timestamp) throws Exception { + if (capture == null) { + super.setTimestamp(timestamp); + } else { + if (!capture.set(CAP_PROP_POS_MSEC, timestamp/1000.0)) { + throw new Exception("set() Error: Could not set CAP_PROP_POS_MSEC to " + timestamp/1000.0 + "."); + } + } + } + + @Override public int getLengthInFrames() { + return capture == null ? super.getLengthInFrames() : + (int)capture.get(CAP_PROP_FRAME_COUNT); + } + @Override public long getLengthInTime() { + return Math.round(getLengthInFrames() * 1000000L / getFrameRate()); + } + + public double getOption(int propId) { + if (capture != null) { + return capture.get(propId); + } + return Double.parseDouble(options.get(Integer.toString(propId))); + } + + /** + * + * @param propId Property ID, look at opencv_videoio for possible values + * @param value + */ + public void setOption(int propId, double value) { + options.put(Integer.toString(propId), Double.toString(value)); + if (capture != null) { + capture.set(propId, value); + } + } + + public void start() throws Exception { + if (filename != null && filename.length() > 0) { + if (apiPreference > 0) { + capture = new VideoCapture(filename, apiPreference); + } else { + capture = new VideoCapture(filename); + } + } else { + capture = new VideoCapture(deviceNumber); + } + + if (format != null && format.length() >= 4) { + format = format.toUpperCase(); + byte cc0 = (byte)format.charAt(0); + byte cc1 = (byte)format.charAt(1); + byte cc2 = (byte)format.charAt(2); + byte cc3 = (byte)format.charAt(3); + capture.set(CAP_PROP_FOURCC, VideoWriter.fourcc(cc0, cc1, cc2, cc3)); + } + + if (imageWidth > 0) { + if (!capture.set(CAP_PROP_FRAME_WIDTH, imageWidth)) { + capture.set(CAP_PROP_FRAME_WIDTH, imageWidth); + } + } + if (imageHeight > 0) { + if (!capture.set(CAP_PROP_FRAME_HEIGHT, imageHeight)) { + capture.set(CAP_PROP_FRAME_HEIGHT, imageHeight); + } + } + if (frameRate > 0) { + capture.set(CAP_PROP_FPS, frameRate); + } + if (bpp > 0) { + capture.set(CAP_PROP_FORMAT, bpp); // ?? + } + if (imageMode == ImageMode.RAW) { + capture.set(CAP_PROP_CONVERT_RGB, 0); + } + + for (Entry e : options.entrySet()) { + capture.set(Integer.parseInt(e.getKey()), Double.parseDouble(e.getValue())); + } + + Mat mat = new Mat(); + + try { + // Before retrieve() starts returning something else then null + // QTKit sometimes requires some "warm-up" time for some reason... + // The first frame on Linux is sometimes null as well, + // so it's probably a good idea to run this for all platforms... ? + int count = 0; + while (count++ < 100 && !capture.read(mat)) { + Thread.sleep(100); + } + } catch (InterruptedException ex) { + // reset interrupt to be nice + Thread.currentThread().interrupt(); + } + if (!capture.read(mat)) { + throw new Exception("read() Error: Could not read frame in start()."); + } + + if (!triggerMode) { + if (!capture.grab()) { + throw new Exception("grab() Error: Could not grab frame. (Has start() been called?)"); + } + } + } + + public void stop() throws Exception { + if (capture != null) { + capture.release(); + capture = null; + } + } + + public void trigger() throws Exception { + Mat mat = new Mat(); + for (int i = 0; i < numBuffers+1; i++) { + capture.read(mat); + } + if (!capture.grab()) { + throw new Exception("grab() Error: Could not grab frame. (Has start() been called?)"); + } + } + + public Frame grab() throws Exception { + if (!capture.retrieve(mat)) { + throw new Exception("retrieve() Error: Could not retrieve frame. (Has start() been called?)"); + } + if (!triggerMode) { + if (!capture.grab()) { + throw new Exception("grab() Error: Could not grab frame. (Has start() been called?)"); + } + } + + if (imageMode == ImageMode.GRAY && mat.channels() > 1) { + if (returnMatrix == null) { + returnMatrix = new Mat(mat.rows(), mat.cols(), mat.depth(), 1); + } + + cvtColor(mat, returnMatrix, CV_BGR2GRAY); + } else if (imageMode == ImageMode.COLOR && mat.channels() == 1) { + if (returnMatrix == null) { + returnMatrix = new Mat(mat.rows(), mat.cols(), mat.depth(), 3); + } + cvtColor(mat, returnMatrix, CV_GRAY2BGR); + } else { + returnMatrix = mat; + } + return converter.convert(returnMatrix); + } +} \ No newline at end of file diff --git a/src/main/java/org/bytedeco/javacv/Seekable.java b/src/main/java/org/bytedeco/javacv/Seekable.java new file mode 100644 index 00000000..de550d79 --- /dev/null +++ b/src/main/java/org/bytedeco/javacv/Seekable.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 Sven Vorlauf + * + * Licensed either under the Apache License, Version 2.0, or (at your option) + * under the terms of the GNU General Public License as published by + * the Free Software Foundation (subject to the "Classpath" exception), + * either version 2, or any later version (collectively, the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.gnu.org/licenses/ + * http://www.gnu.org/software/classpath/license.html + * + * or as provided in the LICENSE.txt file that accompanied this code. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.bytedeco.javacv; + +public interface Seekable { + + public void seek(long offset, int whence); +} \ No newline at end of file diff --git a/src/main/java/us/ihmc/robotDataLogger/logger/MagewellVideoDataLogger.java b/src/main/java/us/ihmc/robotDataLogger/logger/MagewellVideoDataLogger.java index c9f5805a..2aa6716e 100644 --- a/src/main/java/us/ihmc/robotDataLogger/logger/MagewellVideoDataLogger.java +++ b/src/main/java/us/ihmc/robotDataLogger/logger/MagewellVideoDataLogger.java @@ -67,7 +67,7 @@ private void createCaptureInterface() { startCapture(); } - catch (InterruptedException | IOException e) + catch (Exception e) { LogTools.error("Last frame is bad for {} but who cares, shutting down gracefully because of threading", deviceNumber); } @@ -95,7 +95,7 @@ private void createCaptureInterface() } } - public void startCapture() throws IOException, InterruptedException + public void startCapture() throws Exception { grabber.start(); magewellMuxer.start(); From 678bd8f5ebe74ef3458d1eac40b0cb4f107da21b Mon Sep 17 00:00:00 2001 From: Dexton Anderson Date: Fri, 1 Nov 2024 11:10:11 -0500 Subject: [PATCH 2/2] Add openblas dependency --- build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 7d21eba2..800ce603 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -59,6 +59,11 @@ mainDependencies { api("org.bytedeco:ffmpeg:$ffmpegVersion:linux-x86_64") api("org.bytedeco:ffmpeg:$ffmpegVersion:linux-arm64") api("org.bytedeco:ffmpeg:$ffmpegVersion:windows-x86_64") + val openblasVersion = "0.3.23-1.5.9" // TODO: This can probably be removed in 1.5.10 + api("org.bytedeco:openblas:$openblasVersion") + api("org.bytedeco:openblas:$openblasVersion:linux-x86_64") + api("org.bytedeco:openblas:$openblasVersion:linux-arm64") + api("org.bytedeco:openblas:$openblasVersion:windows-x86_64") api("org.freedesktop.gstreamer:gst1-java-core:1.4.0")