start working at inference.h with numcpp

Reviewed-on: #1

.gitignore

@@ -0,0 +1,11 @@
+# .gitignore
+
+
+*~
+*a
+*o
+*#
+.#*
+*rej
+*orig
+pinentry*

CMakeLists.txt

@@ -0,0 +1,58 @@
+cmake_minimum_required(VERSION 3.15)
+project(NCNNTest LANGUAGES CXX)
+
+# Find system packages
+find_package(X11 REQUIRED)
+
+# Set thirdparty directory
+set(THIRDPARTY_DIR ${CMAKE_SOURCE_DIR}/thirdparty)
+
+# Clone minimal dependencies
+if(NOT EXISTS "${THIRDPARTY_DIR}/ncnn")
+    find_package(Git REQUIRED)
+    execute_process(
+        COMMAND ${GIT_EXECUTABLE} clone --depth 1
+            https://github.com/Tencent/ncnn.git
+        WORKING_DIRECTORY ${THIRDPARTY_DIR}
+    )
+endif()
+
+if(NOT EXISTS "${THIRDPARTY_DIR}/NumCpp")
+    find_package(Git REQUIRED)
+    execute_process(
+        COMMAND ${GIT_EXECUTABLE} clone --depth 1
+            https://github.com/dpilger26/NumCpp.git
+        WORKING_DIRECTORY ${THIRDPARTY_DIR}
+    )
+endif()
+
+# Configure ncnn
+set(NCNN_OPENMP OFF CACHE BOOL "Disable OpenMP")
+set(NCNN_VULKAN OFF CACHE BOOL "Disable Vulkan") 
+add_subdirectory(thirdparty/ncnn)
+
+# Configure NumCpp without Boost
+add_library(numcpp INTERFACE)
+target_compile_definitions(numcpp INTERFACE NUMCPP_NO_USE_BOOST)
+target_include_directories(numcpp INTERFACE 
+    ${THIRDPARTY_DIR}/NumCpp/include
+)
+
+# Main executable
+add_executable(ncnn_test 
+    main.cpp
+    capture.cpp
+)
+
+# Link dependencies
+target_link_libraries(ncnn_test PRIVATE 
+    ncnn
+    ${X11_LIBRARIES}
+    numcpp
+)
+
+# C++ standard
+set_target_properties(ncnn_test PROPERTIES
+    CXX_STANDARD 17
+    CXX_STANDARD_REQUIRED ON
+)

README.md

@@ -4,3 +4,24 @@ a ocr inspired by watamote, meant to be independent and free from bloat
 
 <img src="https://files.catbox.moe/5zxbp7.gif" width="30%" />
 
+## build instructions:
+```shell
+cmake -B build -S .
+cmake --build build -j16
+```
+to test you can just run:
+```shell
+./build/ncnn_test
+```
+
+it will show:
+```shell
+Loaded image: 970x1400 channels: 3
+Image checksum: 7.58284e+08
+```
+
+
+
+## screenshot function showcase:
+
+<img src="misc/showcase.gif" alt="Looping 60fps GIF"  width="70%"/>

capture.cpp

@@ -0,0 +1,92 @@
+// Copyright © 2025 Hashirama Senju
+
+#include <X11/Xlib.h>
+#include <X11/Xutil.h>
+#include <X11/cursorfont.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+
+typedef struct {
+    uint8_t* pixels;  // Raw RGB bytes (PPM P6 format)
+    int width;
+    int height;
+    size_t size;      // Total buffer size (header + pixels)
+} PPMBuffer;
+
+// Interactive region selection and capture
+PPMBuffer capture_ppm_region() {
+    PPMBuffer buf = {0};
+    Display* disp = XOpenDisplay(NULL);
+    if (!disp) return buf;
+
+    // Region selection logic
+    Window root = DefaultRootWindow(disp);
+    Cursor cursor = XCreateFontCursor(disp, XC_crosshair);
+    XGrabPointer(disp, root, False, ButtonPressMask|ButtonReleaseMask|PointerMotionMask,
+                GrabModeAsync, GrabModeAsync, root, cursor, CurrentTime);
+
+    XEvent ev;
+    int start_x = 0, start_y = 0, end_x = 0, end_y = 0;
+    int selecting = 0;
+
+    while (1) {
+        XNextEvent(disp, &ev);
+        if (ev.type == ButtonPress) {
+            start_x = ev.xbutton.x;
+            start_y = ev.xbutton.y;
+            selecting = 1;
+        } else if (ev.type == MotionNotify && selecting) {
+            end_x = ev.xmotion.x;
+            end_y = ev.xmotion.y;
+        } else if (ev.type == ButtonRelease) {
+            break;
+        }
+    }
+
+    XUngrabPointer(disp, CurrentTime);
+    XFreeCursor(disp, cursor);
+
+    // Calculate region bounds
+    int x = start_x < end_x ? start_x : end_x;
+    int y = start_y < end_y ? start_y : end_y;
+    int width = abs(end_x - start_x);
+    int height = abs(end_y - start_y);
+
+    // Capture pixels
+    XImage* ximg = XGetImage(disp, root, x, y, width, height, AllPlanes, ZPixmap);
+    if (!ximg) {
+        XCloseDisplay(disp);
+        return buf;
+    }
+
+    // Create PPM buffer
+    const char header[64] = "";
+    int header_len = snprintf((char*)header, sizeof(header),
+                          "P6\n%d %d\n255\n", width, height);
+    
+    buf.width = width;
+    buf.height = height;
+    buf.size = header_len + (width * height * 3);
+    buf.pixels = static_cast<uint8_t*>(malloc(buf.size));
+    
+    // Copy header
+    memcpy(buf.pixels, header, header_len);
+    
+    // Convert XImage to RGB24
+    uint8_t* pixel_ptr = buf.pixels + header_len;
+    for (int y = 0; y < height; y++) {
+        for (int x = 0; x < width; x++) {
+            unsigned long pixel = XGetPixel(ximg, x, y);
+            *pixel_ptr++ = (pixel >> 16) & 0xFF;  // R
+            *pixel_ptr++ = (pixel >> 8) & 0xFF;   // G
+            *pixel_ptr++ = pixel & 0xFF;          // B
+        }
+    }
+
+    // Cleanup
+    XDestroyImage(ximg);
+    XCloseDisplay(disp);
+    return buf;
+}

capture.h

@@ -0,0 +1,17 @@
+// Copyright © 2025 Hashirama Senju
+
+#ifndef CAPTURE_H
+#define CAPTURE_H
+
+#include <cstdint>
+
+typedef struct {
+    uint8_t* pixels;
+    int width;
+    int height;
+    size_t size;
+} PPMBuffer;
+
+PPMBuffer capture_ppm_region();
+
+#endif

inference.h

@@ -0,0 +1,86 @@
+#ifndef INFERENCE_H
+#define INFERENCE_H
+
+#include <ncnn/net.h>
+#include <NumCpp.hpp>
+#include <string>
+#include <algorithm> 
+
+namespace nc = NumCpp;
+
+struct InferenceResult {
+    nc::NdArray<float> logits;
+    nc::NdArray<int64_t> token_ids;
+};
+
+
+nc::NdArray<int64_t> argmax(const nc::NdArray<float>& array) {
+    nc::NdArray<int64_t> result(array.numRows(), 1);
+    
+    for(nc::uint32 row = 0; row < array.numRows(); ++row) {
+        auto begin = array.cbegin(row);
+        auto end = array.cend(row);
+        result[row] = std::distance(begin, std::max_element(begin, end));
+    }
+    
+    return result;
+}
+
+
+
+ncnn::Mat preprocess(const ncnn::Mat& input) {
+    // Resize and normalize
+    ncnn::Mat processed;
+    ncnn::Mat::from_pixels_resize(input, ncnn::Mat::PIXEL_RGB,
+                                input.w, input.h, 224, 224);
+    const float mean[3] = {0.5f, 0.5f, 0.5f};
+    const float norm[3] = {1/255.0f, 1/255.0f, 1/255.0f};
+    processed.substract_mean_normalize(mean, norm);
+    return processed;
+}
+
+nc::NdArray<int64_t> create_sequence_input() {
+    // NumCpp equivalent of np.full((300, 1), 0, dtype=np.int64)
+    return nc::zeros<int64_t>(nc::Shape(300, 1));
+}
+
+InferenceResult run_inference(const ncnn::Mat& input, 
+                            const std::string& model_path = "model") {
+    static ncnn::Net net;
+    net.opt.num_threads = 16;
+    net.opt.use_vulkan_compute = false;
+    
+    // Load model
+    if(net.load_param((model_path + "/ncnn_model.param").c_str()) || 
+       net.load_model((model_path + "/ncnn_model.bin").c_str())) {
+        throw std::runtime_error("Failed to load model");
+    }
+
+    ncnn::Extractor ex = net.create_extractor();
+
+    // Convert NumCpp array to ncnn mat
+    auto sequence_nc = create_sequence_input();
+    ncnn::Mat sequence_mat(sequence_nc.numCols(), sequence_nc.numRows());
+    std::memcpy(sequence_mat.data, sequence_nc.data(), 
+               sequence_nc.size() * sizeof(int64_t));
+
+    // Run inference
+    ex.input("in0", input);
+    ex.input("in1", sequence_mat);
+
+    ncnn::Mat out_mat;
+    if(ex.extract("out0", out_mat) != 0) {
+        throw std::runtime_error("Inference failed");
+    }
+
+    // Convert ncnn mat to NumCpp array
+    nc::NdArray<float> logits(out_mat.h, out_mat.w);
+    std::memcpy(logits.data(), out_mat.data, logits.size() * sizeof(float));
+
+    // Argmax using NumCpp
+    auto token_ids = nc::argmax(logits, nc::Axis::ROW);
+
+    return {logits, token_ids};
+}
+
+#endif // INFERENCE_H

inference_test.py

@@ -1,83 +0,0 @@
-""" 
----------------------------------------------------------------
-Minimal Python Script for Testing Manga-OCR Model (NCNN Version)
-© All Rights Reserved by MW Enterprise and Affiliates
-
-Credits:
-Model conversion courtesy of AJATiX
-
-漫画OCR模型测试脚本(NCNN转换版)
-腾讯模型转换 - 薄荷糖企业及关联公司版权所有
-
-致谢:
-模型转换由AJATiX提供
----------------------------------------------------------------
-
-
-Development Roadmap:
-
-    Implement our own token decoder (dependency reduction)
-
-    Optimize image preprocessing to get better results
-
-    C++ migration with cross-platform clipboard image detection"
-
-"""
-import numpy as np
-import ncnn
-import cv2
-import re
-import jaconv
-from transformers import AutoTokenizer
-
-
-def preprocess_image(image_path):
-    img = cv2.imread(image_path)
-    mat_in = ncnn.Mat.from_pixels_resize(
-        img, ncnn.Mat.PixelType.PIXEL_RGB, img.shape[1], img.shape[0], 224, 224
-    )
-    mean_vals, norm_vals = [0.5] * 3, [1 / 255.0] * 3
-    mat_in.substract_mean_normalize(mean_vals, norm_vals)
-    return mat_in
-
-
-def run_inference(mat_in, model_path="model"):
-    net = ncnn.Net()
-    net.opt.num_threads = 16
-    net.opt.use_vulkan_compute = False
-    net.load_param(f"{model_path}/ncnn_model.param")
-    net.load_model(f"{model_path}/ncnn_model.bin")
-
-    ex = net.create_extractor()
-    ex.input("in0", mat_in)
-    ex.input("in1", ncnn.Mat(np.full((300, 1), 0, dtype=np.int64)))
-
-    try:
-        ret, out0 = ex.extract("out0")
-        print("Output shape:", out0.shape)
-        return np.array(out0)
-    finally:
-        del ex, net
-
-
-def decode_tokens(logits, tokenizer_path="model"):
-    tkz = AutoTokenizer.from_pretrained(tokenizer_path)
-    token_ids = np.argmax(logits, axis=-1)
-    return tkz.decode(token_ids, skip_special_tokens=True)
-
-
-def post_process(text):
-    text = "".join(text.split()).replace("…", "...")
-    text = re.sub("[・.]{2,}", lambda x: (x.end() - x.start()) * ".", text)
-    return jaconv.h2z(text, ascii=True, digit=True)
-
-
-def main(image_path):
-    mat_in = preprocess_image(image_path)
-    logits = run_inference(mat_in)
-    raw_text = decode_tokens(logits)
-    return post_process(raw_text)
-
-
-if __name__ == "__main__":
-    print("PROCESSED OUT:", main("./manga_sample.jpg"))

main.cpp

@@ -0,0 +1,57 @@
+#include <iostream>
+#include <ncnn/net.h>
+
+#define STB_IMAGE_IMPLEMENTATION
+#include "stb_image.h"
+#include "capture.h"  
+
+int main() {
+    // Capture screen region to PPM buffer
+    PPMBuffer ppm = capture_ppm_region();
+    
+    if(!ppm.pixels || ppm.size == 0) {
+        std::cerr << "Failed to capture screen region" << std::endl;
+        return -1;
+    }
+
+    // Load from PPM memory buffer
+    int width, height, channels;
+    unsigned char* image_data = stbi_load_from_memory(
+        ppm.pixels,
+        ppm.size,
+        &width, &height,
+        &channels,
+        3  // Force RGB output
+    );
+
+    if(!image_data) {
+        std::cerr << "Failed to decode PPM: " << stbi_failure_reason() << std::endl;
+        free(ppm.pixels);
+        return -1;
+    }
+
+    std::cout << "Captured image: " 
+              << width << "x" << height 
+              << " channels: " << channels << std::endl;
+
+    // Convert to ncnn Mat (always RGB now)
+    ncnn::Mat ncnn_mat = ncnn::Mat::from_pixels(
+        image_data,
+        ncnn::Mat::PIXEL_RGB,  // Now guaranteed to be RGB
+        width,
+        height
+    );
+
+    // Cleanup
+    stbi_image_free(image_data);
+    free(ppm.pixels);
+
+    // Simple checksum
+    float sum = 0.f;
+    for(int i = 0; i < ncnn_mat.total(); i++) {
+        sum += ncnn_mat[i];
+    }
+    std::cout << "Image checksum: " << sum << std::endl;
+
+    return 0;
+}

silly_decoder.cpp

@@ -1,68 +0,0 @@
-#include <iostream>
-#include <fstream>
-#include <vector>
-#include <string>
-#include <sstream>
-
-// Function that takes a list of indices and returns a single string 
-// concatenating the corresponding vocab entries.
-std::string processIndices(
-    const std::vector<int>& indices,
-    const std::vector<std::pair<int, std::string>>& vocab
-) {
-    std::ostringstream oss;
-    for (size_t i = 0; i < indices.size(); ++i) {
-        int idx = indices[i];
-        if (idx > 0 && idx <= static_cast<int>(vocab.size())) {
-            oss << vocab[idx - 1].second;
-        } else {
-            oss << "[Index " << idx << " out of range]";
-        }
-        
-        if (i != indices.size() - 1)
-            oss << "";
-    }
-    return oss.str();
-}
-
-int main(int argc, char* argv[]) {
-    if (argc < 2) {
-        std::cerr << "Usage: " << argv[0] << " index1,index2,...\n";
-        return 1;
-    }
-
-    std::ifstream file("vocab.txt");
-    if (!file) {
-        std::cerr << "Error opening file.\n";
-        return 1;
-    }
-
-    // Read vocab.txt and assign indices starting from 1.
-    std::vector<std::pair<int, std::string>> vocab;
-    std::string line;
-    int index = 1;
-    while (std::getline(file, line)) {
-        vocab.emplace_back(index++, line);
-    }
-    file.close();
-
-    // Convert CLI input into a list of indices.
-    std::vector<int> indices;
-    std::istringstream input(argv[1]);
-    std::string token;
-    while (std::getline(input, token, ',')) {
-        try {
-            indices.push_back(std::stoi(token));
-        } catch (const std::exception&) {
-            std::cerr << "Invalid index: " << token << '\n';
-        }
-    }
-
-    // Process the indices and retrieve the aggregated result as a single string.
-    std::string result = processIndices(indices, vocab);
-
-
-    std::cout << "Result: " << result << '\n';
-
-    return 0;
-}