Skip to content

Commit

Permalink
Debugging for Android 9, see #2
Browse files Browse the repository at this point in the history
  • Loading branch information
@sergeivolodin
sergeivolodin committed May 10, 2021
1 parent f684d01 commit dc99ca8
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 211 deletions.
255 changes: 46 additions & 209 deletions src/imagebackend/nv21videofilterrunnable.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
// Qt includes
#include <QDebug>
#include <QImage>
#include <QString>
#include <QOpenGLContext>

// C functions
Expand Down Expand Up @@ -115,6 +116,9 @@ QVideoFrame NV21VideoFilterRunnable::run(QVideoFrame *inputFrame)

// obtaining current GL family
auto version(context->isOpenGLES() ? "#version 300 es\n" : "#version 130\n");
qDebug() << "gl version " << version;



// see https://github.com/aseba-community/thymio-ar/blob/eb942a5e96761303512a07dc6e5057749ff8939e/vision-video-filter.h
// this is for averaging the image
Expand Down Expand Up @@ -151,119 +155,25 @@ QVideoFrame NV21VideoFilterRunnable::run(QVideoFrame *inputFrame)
in lowp vec2 uvBase;
uniform sampler2D image;
const uint sampleByPixel = %1u;
const lowp vec2 uvDelta = vec2(%2, %3);
// want this mean hue (0-1)
uniform lowp float mean_h_;
// maximal deviation in HUE (0-1)
uniform lowp float delta_h_;
// min saturation (0-1)
uniform lowp float min_s_;
// max saturation (0-1)
uniform lowp float max_s_;
// min value (0-1)
uniform lowp float min_v_;
// max value (0-1)
uniform lowp float max_v_;
// marker X points
uniform lowp vec4 marker_x;
// marker X points
uniform lowp vec4 marker_y;
// output RGBA
out lowp vec4 fragment;
// convert RGB to HSV, see https://gist.github.com/msbarry/cd98f928542f5152111a
lowp vec3 rgb2hsv(lowp vec3 c)
{
lowp vec4 K = vec4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0);
lowp vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g));
lowp vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r));
lowp float d = q.x - min(q.w, q.y);
lowp float e = 1.0e-10;
return vec3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x);
}
// check if point is inside a polygon https://stackoverflow.com/questions/11716268/point-in-polygon-algorithm
bool pnpoly(lowp vec4 vertx, lowp vec4 verty, lowp float testx, lowp float testy)
{
const int nvert = 4;
int i, j;
bool c = false;
for (i = 0, j = nvert-1; i < nvert; j = i++) {
if ( ((verty[i]>testy) != (verty[j]>testy)) &&
(testx < (vertx[j]-vertx[i]) * (testy-verty[i]) / (verty[j]-verty[i]) + vertx[i]) )
c = !c;
}
return c;
}
// main function
void main(void) {
// sum of input pixels for this output pixels
lowp vec3 sum = vec3(0.0, 0.0, 0.0);
// summing up the input pixels
for (uint x = 0u; x < sampleByPixel; ++x) {
for (uint y = 0u; y < sampleByPixel; ++y) {
lowp vec2 uv = uvBase + vec2(x, y) * uvDelta;
sum += texture(image, uv).bgr;
}
}
// dividing by the number of summands
lowp float divisor = float(sampleByPixel * sampleByPixel);
lowp vec3 rgb = vec3(
sum.b / divisor,
sum.g / divisor,
sum.r / divisor);
// obtaining HSV data
lowp vec3 hsv = rgb2hsv(rgb);
lowp float h = hsv.r;
lowp float s = hsv.g;
lowp float v = hsv.b;
// difference in HUE (0-1)
lowp float diff;
// calculating shortest distance between angles
if(h > 0.0) { diff = h - mean_h_; }
else { diff = mean_h_ - h; }
if(0.5 - diff < diff) { diff = 0.5 - diff; }
// output: 1.0 or 0.0
lowp float value;
// doing the thresholding
if(s >= min_s_ && s <= max_s_ &&
v >= min_v_ && v <= max_v_ &&
diff < delta_h_) { value = 1.0; }
else value = 0.0;
// blackening output if outside marker
if(!pnpoly(marker_x, marker_y, uvBase.x, uvBase.y)) {
value = 0.0;
}
// output: white or black
fragment = vec4(value, value, value, 1.0);
fragment = vec4(100., 0.2, 0.3, 1.0);
}
)";

// creating vertex shader
program.addShaderFromSourceCode(QOpenGLShader::Vertex, vertex);

// creating fragment shader
program.addShaderFromSourceCode(QOpenGLShader::Fragment, fragment.
arg(sampleByPixelI).
arg(uvDelta.x()).
arg(uvDelta.y()));
program.addShaderFromSourceCode(QOpenGLShader::Fragment, fragment);
// arg(sampleByPixelI).
// arg(uvDelta.x()).
// arg(uvDelta.y()));

// linking the shader
program.link();
Expand All @@ -272,14 +182,14 @@ QVideoFrame NV21VideoFilterRunnable::run(QVideoFrame *inputFrame)
imageLocation = program.uniformLocation("image");

// obtaining shader uniform locations
mean_h_ = program.uniformLocation("mean_h_");
delta_h_ = program.uniformLocation("delta_h_");
min_s_ = program.uniformLocation("min_s_");
max_s_ = program.uniformLocation("max_s_");
min_v_ = program.uniformLocation("min_v_");
max_v_ = program.uniformLocation("max_v_");
marker_x_ = program.uniformLocation("marker_x");
marker_y_ = program.uniformLocation("marker_y");
// mean_h_ = program.uniformLocation("mean_h_");
// delta_h_ = program.uniformLocation("delta_h_");
// min_s_ = program.uniformLocation("min_s_");
// max_s_ = program.uniformLocation("max_s_");
// min_v_ = program.uniformLocation("min_v_");
// max_v_ = program.uniformLocation("max_v_");
// marker_x_ = program.uniformLocation("marker_x");
// marker_y_ = program.uniformLocation("marker_y");

// creating output framebuffer
out_fbo = new QOpenGLFramebufferObject(outputWidth, outputHeight);
Expand All @@ -289,57 +199,13 @@ QVideoFrame NV21VideoFilterRunnable::run(QVideoFrame *inputFrame)

// attaching the created texture as to the framebuffer (texture will contain output RGBA data)
gl->glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, outTex, 0); GL_CHECK_ERROR();

#ifdef USEGRAPHICBUFFER
graphicBuf = new GraphicBuffer(outputWidth, outputHeight, PixelFormat::PIXEL_FORMAT_RGBA_8888,
GraphicBuffer::USAGE_SW_READ_OFTEN | GraphicBuffer::USAGE_SW_WRITE_NEVER |
GraphicBuffer::USAGE_HW_RENDER);
#else
// filling in the usage for HardwareBuffer
usage.format = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
usage.height = outputHeight;
usage.width = outputWidth;
usage.layers = 1;
usage.rfu0 = 0;
usage.rfu1 = 0;
usage.stride = 10;
usage.usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER
| AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT;
#endif

#ifdef USEGRAPHICBUFFER
clientBuf = graphicBuf->getNativeBuffer();
#else
// creating an Android 8 HardwareBuffer
error = AHardwareBuffer_allocate(&usage, &graphicBuf); HWB_CHECK_ERROR();

// obtaining buffer description -> can get real stride in the code which follows
AHardwareBuffer_describe(graphicBuf, &usage1);

// obtaining native buffer
clientBuf = eglGetNativeClientBufferANDROID(graphicBuf); EGL_CHECK_ERROR();
#endif

#ifdef DEBUG_SHADER
qDebug() << "GraphicBuf" << graphicBuf << "clientBuf" << clientBuf;
#endif

// obtaining the EGL display
disp = eglGetDisplay(EGL_DEFAULT_DISPLAY); EGL_CHECK_ERROR();

// specifying the image attributes
EGLint eglImageAttributes[] = {EGL_IMAGE_PRESERVED_KHR, EGL_TRUE, EGL_NONE};

// creating an EGL image
imageEGL = eglCreateImageKHR(disp, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, clientBuf, eglImageAttributes);
EGL_CHECK_ERROR();
}

//setting current texture
gl->glActiveTexture(GL_TEXTURE0); GL_CHECK_ERROR();

// binding INPUT texture
gl->glBindTexture(QOpenGLTexture::Target2D, inputFrame->handle().toUInt());
gl->glBindTexture(QOpenGLTexture::Target2D, inputFrame->handle().toUInt()); GL_CHECK_ERROR();

// specifying INPUT texture parameters
gl->glTexParameteri(QOpenGLTexture::Target2D, QOpenGLTexture::DirectionS, QOpenGLTexture::ClampToEdge); GL_CHECK_ERROR();
Expand All @@ -352,25 +218,25 @@ QVideoFrame NV21VideoFilterRunnable::run(QVideoFrame *inputFrame)
program.enableAttributeArray(0);

// configuring the shader: setting hue threshold parameters
program.setUniformValue(mean_h_, GLfloat(mean_h));
program.setUniformValue(delta_h_, GLfloat(delta_h));
program.setUniformValue(min_s_, GLfloat(min_s));
program.setUniformValue(max_s_, GLfloat(max_s));
program.setUniformValue(min_v_, GLfloat(min_v));
program.setUniformValue(max_v_, GLfloat(max_v));

// setting marker location (blackening everything else)
if(marker.size() == 4)
{
program.setUniformValue(marker_x_, marker.at(0).x(), marker.at(1).x(), marker.at(2).x(), marker.at(3).x());
program.setUniformValue(marker_y_, marker.at(0).y(), marker.at(1).y(), marker.at(2).y(), marker.at(3).y());
}

// otherwise, showing all
else {
program.setUniformValue(marker_x_, 0, 1, 1, 0);
program.setUniformValue(marker_y_, 0, 0, 1, 1);
}
// program.setUniformValue(mean_h_, GLfloat(mean_h));
// program.setUniformValue(delta_h_, GLfloat(delta_h));
// program.setUniformValue(min_s_, GLfloat(min_s));
// program.setUniformValue(max_s_, GLfloat(max_s));
// program.setUniformValue(min_v_, GLfloat(min_v));
// program.setUniformValue(max_v_, GLfloat(max_v));

// // setting marker location (blackening everything else)
// if(marker.size() == 4)
// {
// program.setUniformValue(marker_x_, marker.at(0).x(), marker.at(1).x(), marker.at(2).x(), marker.at(3).x());
// program.setUniformValue(marker_y_, marker.at(0).y(), marker.at(1).y(), marker.at(2).y(), marker.at(3).y());
// }

// // otherwise, showing all
// else {
// program.setUniformValue(marker_x_, 0, 1, 1, 0);
// program.setUniformValue(marker_y_, 0, 0, 1, 1);
// }

// binding OUTPUT framebuffer
gl->glBindFramebuffer(GL_FRAMEBUFFER, out_fbo->handle()); GL_CHECK_ERROR();
Expand All @@ -384,49 +250,17 @@ QVideoFrame NV21VideoFilterRunnable::run(QVideoFrame *inputFrame)
// binding the OUTPUT texture
gl->glBindTexture(GL_TEXTURE_2D, outTex); GL_CHECK_ERROR();

// attaching an EGLImage to OUTPUT texture
glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, imageEGL); EGL_CHECK_ERROR();

// finish all draw calls
gl->glFinish(); GL_CHECK_ERROR();


// drawing output image on input frame, if requested
if(show_processed) gl->glCopyImageSubData(outTex, GL_TEXTURE_2D, 0, 0, 0, 0,
// if(show_processed)
gl->glCopyImageSubData(outTex, GL_TEXTURE_2D, 0, 0, 0, 0,
inputFrame->handle().toUInt(), GL_TEXTURE_2D,
0, 0, 0, 0, outputWidth, outputHeight, 1);

// pointer to READ from EGLImage
unsigned char *readPtr;

// pointer to WRITE the image (in memory)
unsigned char *writePtr = readBuffer;

// locking the buffer
#ifdef USEGRAPHICBUFFER
error = graphicBuf->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, (void**) &readPtr);
#else
error = AHardwareBuffer_lock(graphicBuf, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, nullptr, (void**) &readPtr);
#endif
HWB_CHECK_ERROR();

// obtaining real stride
#ifdef USEGRAPHICBUFFER
unsigned int stride = graphicBuf->getStride();
#else
unsigned int stride = usage1.stride;
#endif

// copying all rows from EGLImage to readBuffer
for (int row = 0; row < outputHeight; row++) {
memcpy(writePtr, readPtr, outputWidth * 4);
readPtr = (unsigned char *)(int(readPtr) + stride * 4);
writePtr = (unsigned char *)(int(writePtr) + outputWidth * 4);
}

// unlocking the buffer
#ifdef USEGRAPHICBUFFER
error = graphicBuf->unlock();
#else
error = AHardwareBuffer_unlock(graphicBuf, nullptr); HWB_CHECK_ERROR();
#endif

// wrapping the buffer inside a QImage
image = QImage(readBuffer, outputWidth, outputHeight, QImage::Format_RGBA8888);

Expand All @@ -437,6 +271,9 @@ QVideoFrame NV21VideoFilterRunnable::run(QVideoFrame *inputFrame)
// converting the image to Grayscale
image = image.convertToFormat(QImage::Format_Grayscale8);

static int i = 0;
image.save(QString("/storage/emulated/0/shader%1.png").arg(++i));

// sending the black/white image to listeners
emit imageConverted(PipelineContainer<QImage>(image, image_info.checkpointed("Sent")));

Expand Down
4 changes: 2 additions & 2 deletions src/imagebackend/nv21videofilterrunnable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
* @def DEBUG_NV21_FILTER
* @brief If defined, the class will print GL/EGL/HardwareBuffer return codes
*/
//#define DEBUG_NV21_FILTER
#undef DEBUG_NV21_FILTER
#define DEBUG_NV21_FILTER
//#undef DEBUG_NV21_FILTER

class NV21VideoFilter;
class QtCameraBackend;
Expand Down
3 changes: 3 additions & 0 deletions src/imagebackend/qtbackend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ void QtCameraBackend::handleFinished()
// saving to buffer
processQImage(watcher.result());

static int i = 0;
buf.save(QString("/storage/emulated/0/qimage%1.png").arg(++i));

// sending image
emit imageAvailable(PipelineContainer<QImage>
(buf.copy(), image_info.checkpointed("Camera")));
Expand Down

0 comments on commit dc99ca8

Please sign in to comment.