How to process every single frame of a video?

I have succesfully written a simple program inside my main program that exports a video off-screen using Render Targets.
The problem is: my computer sucks. Therefore, the video output sucks as well, with lots of lag and frameskipping.
I want to ensure that the exported video has 100% of the frames exported, even if it takes a long time. I guess this means decoupling video exporting from video playback. I was able to do this in Processing (but it was slooow and it was pretty much a sloppy workaround).

here’s my current simple exporting program

import org.openrndr.application
import org.openrndr.color.ColorRGBa
import org.openrndr.draw.renderTarget
import org.openrndr.draw.isolatedWithTarget
import org.openrndr.ffmpeg.VideoPlayerFFMPEG
import org.openrndr.ffmpeg.VideoWriter
import utils.MediaManager
import utils.MediaType
import java.io.File
import java.util.logging.Logger

fun main() = application {
    configure {
        width = 960
        height = 540
        title = "Off-Screen Video Export with Progress Counter"
    }

    program {
        val logger = Logger.getLogger("SimpleVideoPlayer")
        val mediaManager = MediaManager()
        val videoFile = File("c:\\videos\\godgangs\\playlist\\01.mp4")

        var videoPlayer: VideoPlayerFFMPEG? = null
        val playDuration = 10_000L // 10 seconds in milliseconds
        var startTime: Long? = null
        var videoStopped = false
        var elapsedTime = 0L

        // Set up the video writer
        val videoWriter = VideoWriter()
        videoWriter.size(width, height)
        videoWriter.output("video/output.mp4")
        videoWriter.start()

        // Set up the render target
        val videoTarget = renderTarget(width, height) {
            colorBuffer()
            depthBuffer()
        }

        extend {
            // Render video frames off-screen
            drawer.isolatedWithTarget(videoTarget) {
                clear(ColorRGBa.BLACK)

                if (videoPlayer == null) {
                    logger.info("Loading video: ${videoFile.absolutePath}")
                    videoPlayer = mediaManager.loadMedia(videoFile, MediaType.VIDEO) as? VideoPlayerFFMPEG
                    videoPlayer?.play()
                    startTime = System.currentTimeMillis()
                }

                val currentTime = System.currentTimeMillis()
                startTime?.let {
                    elapsedTime = currentTime - it
                    if (elapsedTime >= playDuration && !videoStopped) {
                        videoPlayer?.pause()
                        videoPlayer?.seek(0.0)
                        videoStopped = true
                        logger.info("Stopping video after 10 seconds")
                    } else if (!videoStopped) {
                        videoPlayer?.draw(drawer, 0.0, 0.0, width.toDouble(), height.toDouble())
                    }
                }

                // Write the frame to the video
                videoWriter.frame(videoTarget.colorBuffer(0))
            }

            // Display a black screen with a progress counter
            drawer.clear(ColorRGBa.BLACK)
            drawer.fill = ColorRGBa.WHITE
            val timeString = String.format("%.1f", elapsedTime / 1000.0)
            drawer.text("Exporting... Time: $timeString s", width / 2.0 - 100.0, height / 2.0)

            // Stop the video writer after the play duration
            if (elapsedTime >= playDuration) {
                videoWriter.stop()
                application.exit()
            }
        }

        ended.listen {
            logger.info("Application ending")
            mediaManager.disposeAll()
        }
    }
}

it uses this MediaManager that handles media throughout my application

package utils

import org.openrndr.draw.ColorBuffer
import org.openrndr.ffmpeg.VideoPlayerFFMPEG
import java.io.File
import java.util.logging.Logger

private val logger = Logger.getLogger(Constants.MEDIA_MANAGER_LOGGER_NAME)

enum class MediaType {
    IMAGE, VIDEO
}

class MediaManager {
    private val loadedImages = mutableMapOf<File, ColorBuffer>()
    private val loadedVideos = mutableMapOf<File, VideoPlayerFFMPEG>()

    fun loadMedia(file: File, mediaType: MediaType): Any? {
        return when (mediaType) {
            MediaType.IMAGE -> loadImage(file)
            MediaType.VIDEO -> loadVideo(file)
        }
    }

    private fun loadImage(file: File): ColorBuffer? {
        return loadedImages[file] ?: tryWithLogging("load image ${file.name}", logger) {
            logger.info("Loading image: ${file.name}")
            val image = org.openrndr.draw.loadImage(file)
            loadedImages[file] = image
            logger.info("Image loaded successfully: ${file.name}")
            image
        }
    }

    private fun loadVideo(file: File): VideoPlayerFFMPEG? {
        return loadedVideos[file] ?: tryWithLogging("load video ${file.name}", logger) {
            logger.info("Loading video: ${file.name}")
            val video = VideoPlayerFFMPEG.fromFile(file.absolutePath)
            video.ended.listen { video.restart() }
            loadedVideos[file] = video
            logger.info("Video loaded successfully: ${file.name}")
            video
        }
    }

    fun disposeAll() {
        loadedVideos.values.forEach { it.dispose() }
        loadedVideos.clear()
        loadedImages.values.forEach { it.destroy() }
        loadedImages.clear()
    }
}

If anybody can point me in the right direction I’ll be glad and thankful!
(Abe Pazos’ photo will be loaded at the starting screen of my app with bells and whistles around him already)

1 Like

Hi! :slight_smile: Short time no see! XD

I think this might help:

val v = VideoPlayerFFMPEG.fromFile(path,
    clock = { frameCount / 60.0 / 30.0 },
    mode = PlayMode.VIDEO,
    configuration = conf
)

The thing behind clock = is a function. The normal approach is clock = { seconds } so the time used for the video is the current time in seconds. By making it depend on frameCount you can calculate which frame you want to display. If the frame rate goes down, the video location should still match.

I don’t know the effect of using conf, but here one based on the source code:

        val conf = VideoPlayerConfiguration()
        conf.let {
            it.videoFrameQueueSize = 5
            it.packetQueueSize = 1250
            it.displayQueueSize = 2
            //it.synchronizeToClock = false
        }

I don’t know if this conf is appropriate in this case. Just leaving it there in case it’s useful. I would first try without any custom configuration.

Looking forward to see what you create with all of this :slight_smile:

1 Like

Thanks again, pal. I tried to incorporate your suggestion, but i have this:

Unresolved reference: PlayMode

Don’t know what I’m missing.

import org.openrndr.application
import org.openrndr.color.ColorRGBa
import org.openrndr.draw.renderTarget
import org.openrndr.draw.isolatedWithTarget
import org.openrndr.ffmpeg.VideoPlayerFFMPEG
import org.openrndr.ffmpeg.VideoWriter
import utils.MediaManager
import utils.MediaType
import java.io.File
import java.util.logging.Logger

fun main() = application {
    configure {
        width = 960
        height = 540
        title = "Off-Screen Video Export with Progress Counter"
    }

    program {
        val logger = Logger.getLogger("SimpleVideoPlayer")
        val mediaManager = MediaManager()
        val videoFile = File("c:\\videos\\godgangs\\playlist\\01.mp4")

        var videoPlayer: VideoPlayerFFMPEG? = null
        val playDuration = 10_000L // 10 seconds in milliseconds
        var startTime: Long? = null
        var videoStopped = false
        var elapsedTime = 0L

        // Set up the video writer
        val videoWriter = VideoWriter()
        videoWriter.size(width, height)
        videoWriter.output("video/output.mp4")
        videoWriter.start()

        // Set up the render target
        val videoTarget = renderTarget(width, height) {
            colorBuffer()
            depthBuffer()
        }

        extend {
            // Render video frames off-screen
            drawer.isolatedWithTarget(videoTarget) {
                clear(ColorRGBa.BLACK)

                if (videoPlayer == null) {
                    logger.info("Loading video: ${videoFile.absolutePath}")
                    videoPlayer = VideoPlayerFFMPEG.fromFile(
                        videoFile.absolutePath,
                        clock = { frameCount / 60.0 / 30.0 }, // Synchronize video with frame count
                        mode = VideoPlayerFFMPEG.PlayMode.VIDEO
                    )
                    videoPlayer?.play()
                    startTime = System.currentTimeMillis()
                }

                val currentTime = System.currentTimeMillis()
                startTime?.let {
                    elapsedTime = currentTime - it
                    if (elapsedTime >= playDuration && !videoStopped) {
                        videoPlayer?.pause()
                        videoPlayer?.seek(0.0)
                        videoStopped = true
                        logger.info("Stopping video after 10 seconds")
                    } else if (!videoStopped) {
                        videoPlayer?.draw(drawer, 0.0, 0.0, width.toDouble(), height.toDouble())
                    }
                }

                // Write the frame to the video
                videoWriter.frame(videoTarget.colorBuffer(0))
            }

            // Display a black screen with a progress counter
            drawer.clear(ColorRGBa.BLACK)
            drawer.fill = ColorRGBa.WHITE
            val timeString = String.format("%.1f", elapsedTime / 1000.0)
            drawer.text("Exporting... Time: $timeString s", width / 2.0 - 100.0, height / 2.0)

            // Stop the video writer after the play duration
            if (elapsedTime >= playDuration) {
                videoWriter.stop()
                application.exit()
            }
        }

        ended.listen {
            logger.info("Application ending")
            mediaManager.disposeAll()
        }
    }
}

import org.openrndr.ffmpeg.PlayMode ?

I hit alt+enter on top of the highlighted word to import it.

yeah, I had already tried that but it doesn’t work :frowning:

1 Like

Did you manage to make it work? I’m not sure if pressing alt+enter didn’t work, or was it the import itself?

Yeah, i don’t remember how did I solve it but I did. In fact I have advanced a bit on my VideoExporter, it is exporting what I want to, but I’m still struggling with exporting video consistently.

package utils

import org.openrndr.Program
import org.openrndr.draw.*
import org.openrndr.ffmpeg.VideoWriter
import models.Project
import org.openrndr.color.ColorRGBa
import renderers.TimelineRenderer
import renderers.drawProgramContents
import java.util.logging.Logger

class VideoExporter(
    private val program: Program,
    private val project: Project,
    private val timelineRenderer: TimelineRenderer,
    private val imageManager: ImageManager,
    private val videoManager: VideoManager
) {
    private val logger = Logger.getLogger(Constants.VIDEO_EXPORTER_LOGGER_NAME)
    private var isExporting = false
    private var progress = 0.0
    private var renderTarget: RenderTarget? = null
    private var videoWriter: VideoWriter? = null
    private var currentFrame = 0
    private var totalFrames = 0
    private lateinit var exportProject: Project
    private val frameRate = 30 // Frames per second

    fun exportVideo(outputFilePath: String) {
        if (isExporting) {
            logger.warning("Export already in progress")
            return
        }

        isExporting = true
        progress = 0.0
        currentFrame = 0

        renderTarget = renderTarget(program.width, program.height) {
            colorBuffer()
            depthBuffer()
        }

        videoWriter = VideoWriter().apply {
            size(program.width, program.height)
            output(outputFilePath)
            frameRate = this@VideoExporter.frameRate // Set the frame rate
            start()
        }

        totalFrames = (project.totalDuration / 1000.0 * frameRate).toInt()
        exportProject = project.copy(playing = true, startTime = 0L)

        logger.info("Starting video export: $outputFilePath")

        // Start the export process
        exportAllFrames()
    }

    private fun exportAllFrames() {
        while (currentFrame < totalFrames) {
            renderExportFrame()
            currentFrame++
            progress = currentFrame.toDouble() / totalFrames
        }
        finishExport()
    }

    private fun renderExportFrame() {
        val currentTime = (currentFrame / frameRate.toDouble()) * 1000.0 // Convert frame to milliseconds
        exportProject.startTime = System.currentTimeMillis() - currentTime.toLong()
        exportProject.currentSceneIndex = exportProject.scenes.indexOfLast { it.startTime <= currentTime }

        renderTarget?.let { rt ->
            program.drawer.isolatedWithTarget(rt) {
                clear(ColorRGBa.TRANSPARENT)
                program.drawProgramContents(exportProject, timelineRenderer, "Exporting...", imageManager, videoManager, isExporting = true)
            }
            videoWriter?.frame(rt.colorBuffer(0))
        }
    }

    private fun finishExport() {
        videoWriter?.stop()
        renderTarget?.destroy()
        renderTarget = null
        videoWriter = null
        isExporting = false
        progress = 1.0
        logger.info("Video export completed")
    }

    fun cancelExport() {
        if (isExporting) {
            videoWriter?.stop()
            renderTarget?.destroy()
            renderTarget = null
            videoWriter = null
            isExporting = false
            progress = 0.0
            logger.info("Video export cancelled")
        }
    }

    fun getProgress(): Double = progress

    fun isExporting(): Boolean = isExporting
}

It uses a MP3 file to set the duration of the playback, then loads images and videos sequentially based on Scene Numbers, which are determined by a timestamped transcript .txt file. (It’s not exporting audio, for now I’m happy with syncing things with FFMpeg later).

Problem is that in the exported video, the videos included in the playback are lagging, with frameskips and weird behavior.

Here’s the video output exported by my OPENRNDR program.
Here’s one of the videos used in the playback.
Here’s another one used in the playback.

I’m trying to find a way to ensure a smooth video export. I’m quite sure if I have a nice PC it would be better, but the point here is to make sure a slow computer exports the same video as a fast computer.

This is a lightweight video editor I’m working on, it will apply shaders and other effects to the playback as someone would apply them to a page using CSS (with a fx.txt text file that applies shaders to scenes).

I had a workaround in mind, which was to merge the original video files in the export process instead of loading them into the program and re-exporting them.

Problem is, this way I won’t be able to apply shaders to the videos, which is the whole point.

Maybe the exporting process can be streamlined in a similar fashion… process the videos one at a time to ensure consistency then merge them back?
I don’t know what benefits can come out of this, I feel I’m missing something.

Just saw this:

Using the Screenshots extension to export individual frames of the visual output to generate the video. Something I may try, only concern is about performance regarding capturing screenshots from video rendered inside OPENRNDR in my machine.
Key would be ensure it takes all the necessary screenshots of a Render Target.

Speculations

I came up with a different approach: avoid using the video player and extract all video frames. Then you can be sure you get them all. Assumes all video files have the same frame rate. This can potentially be faster too, as it saves the frames as fast as possible, detached from the programs frame rate.

import org.openrndr.application
import org.openrndr.color.ColorRGBa
import org.openrndr.draw.*
import org.openrndr.extra.imageFit.FitMethod
import org.openrndr.extra.imageFit.imageFit
import org.openrndr.ffmpeg.VideoWriter
import java.io.File
fun getVideoDimensions(path: File): IntVector2 {
    val pbGetVideoDimensions = ProcessBuilder(
        "ffprobe",
        "-v", "error",
        "-select_streams", "v:0",
        "-show_entries", "stream=width,height",
        "-of", "csv=s=x:p=0",
        path.absolutePath
    )

    val videoDimensions = String(pbGetVideoDimensions.start().inputStream.readAllBytes()).trim().split("x")
    return IntVector2(
        videoDimensions[0].toInt(),
        videoDimensions[1].toInt()
    )
}
/**
 * A class to provide access to all the frames of a video file.
 * It extracts all frames of the [path] video file into a subfolder in PNG format.
 */
class VideoFrames(path: File, private val drawer: Drawer) {
    private val videoDimensions = getVideoDimensions(path)
    private var rt = renderTarget(videoDimensions.x, videoDimensions.y) {
        colorBuffer()
    }

    // A folder with the same name as the video file with ".frames" appended
    private val folder = File(path.absolutePath + ".frames")

    // Don't throw random files into the generated folder, it will mess up
    // the frameCount calculation
    var frameCount = folder.listFiles()?.size ?: 0
        private set

    // Used to avoid regenerating a frame if it was just requested.
    // Useful if we draw the same frame multiple times (aka pause).
    private var requestedFrame = -1

    // Returns a frame loaded from disk, or TRANSPARENT if out of range
    operator fun get(frame: Int): ColorBuffer {
        if (frame != requestedFrame) {
            drawer.isolatedWithTarget(rt) {
                ortho(rt)
                if (frame < 0 || frame >= frameCount) {
                    clear(ColorRGBa.TRANSPARENT)
                } else {
                    val imagePath = "$folder/frame${String.format("%05d", frame + 1)}.png"
                    val loadedImage = loadImage(imagePath)
                    image(loadedImage)
                    loadedImage.destroy()
                }
            }
            requestedFrame = frame
        }
        return rt.colorBuffer(0)
    }

    // Create the frame folder if it doesn't exist
    // and fill it with extracted frames
    init {
        if (!folder.exists()) {
            folder.mkdir()
        }
        if (frameCount == 0) {
            val args = listOf(
                "ffmpeg",
                "-r", "1",
                "-i", path.absolutePath,
                "-r", "1",
                "$folder/frame%05d.png"
            )
            val pb = ProcessBuilder(args)
            pb.start().waitFor()
            frameCount = folder.listFiles()?.size ?: 0
            println("$frameCount frames extracted")
        } else {
            println("$frameCount frames found")
        }
    }

    fun destroy() = rt.destroy()
}
// A sample program making use of VideoFrames.
// It just draws all the frames from in.mp4 into out.mp4, adding a text on top.
fun main() = application {
    program {
        val videoWriter = VideoWriter()
        videoWriter.size(width, height)
        videoWriter.output("/tmp/out.mp4")
        videoWriter.start()

        val canvas = renderTarget(width, height) {
            colorBuffer()
            depthBuffer()
        }

        val frames = VideoFrames(File("/tmp/in.mp4"), drawer)

        val font = loadFont("data/fonts/default.otf", 200.0)

        for (i in 0 until frames.frameCount) {
            val frame = frames[i]
            drawer.isolatedWithTarget(canvas) {
                imageFit(frame, bounds, fitMethod = FitMethod.Cover)
                fill = ColorRGBa.CYAN
                fontMap = font
                text("$i", 10.0, height - 20.0)
            }
            // Save the canvas into the video file
            videoWriter.frame(canvas.colorBuffer(0))
            println("Saved frame $i")
        }
        videoWriter.stop()
        frames.destroy()

        println("Done")

        extend {
        }
    }
}

In the example we play the frames in the exact same order (creating an output matching the input), but we could easily go backwards, or jump to random frames. We could create a video longer or shorter than the original, apply effects, etc.

One has to be careful with the sizes of the produced frames folders, and maybe delete them when done.

Wow, thanks a lot. I will try to incorporate this into my program and I will come back with the results. There are situations that 3 videos are being rendered on screen at the same time, but assuming all are at the same framerate it shouldn’t be a problem. As soon as I have results I will return here. Thanks again.

1 Like

Hi Abe. Coming back to this after some time, I had to prioritize other stuff.
I’m implementing this into my program’s video exporter.

It works perfectly if I give it two passes.
First, I run the program and extract the frames.
Then I close it. When I open it again, it detects the frames in the folder and compiles the video correctly.

I’m trying to make the program know that all the frames were extracted and it should start compiling, but I can’t make it work. I have even used ffprobe to get the framerate and the duration of the video to calculate the total frames, and it works but I just can’t make the program know the frame rate extraction has ended.

If you have any idea that I can try, I’ll be extremely grateful. I’ve worked hours on this before I resorted to you again. Thanks.

    private fun getVideoDuration(path: File): Double {
        logger.info("Getting video duration for file: ${path.absolutePath}")
        val pbGetVideoDuration = ProcessBuilder(
            "ffprobe",
            "-v", "error",
            "-show_entries", "format=duration",
            "-of", "default=noprint_wrappers=1:nokey=1",
            path.absolutePath
        )

        try {
            val process = pbGetVideoDuration.start()
            val output = String(process.inputStream.readAllBytes()).trim()
            val exitCode = process.waitFor()

            if (exitCode != 0) {
                val errorOutput = String(process.errorStream.readAllBytes())
                throw IOException("ffprobe process failed with exit code $exitCode. Error: $errorOutput")
            }

            val duration = output.toDoubleOrNull()
            if (duration == null) {
                throw NumberFormatException("Invalid video duration: $output")
            }

            logger.info("Video duration: $duration seconds")
            return duration
        } catch (e: Exception) {
            logger.log(Level.SEVERE, "Error getting video duration", e)
            throw e
        }
    }

    private fun getFrameRate(path: File): Double {
        logger.info("Getting frame rate for file: ${path.absolutePath}")
        val pbGetFrameRate = ProcessBuilder(
            "ffprobe",
            "-v", "error",
            "-select_streams", "v:0",
            "-count_packets",
            "-show_entries", "stream=nb_read_packets",
            "-of", "csv=p=0",
            path.absolutePath
        )

        try {
            val process = pbGetFrameRate.start()
            val output = String(process.inputStream.readAllBytes()).trim()
            val exitCode = process.waitFor()

            if (exitCode != 0) {
                val errorOutput = String(process.errorStream.readAllBytes())
                throw IOException("ffprobe process failed with exit code $exitCode. Error: $errorOutput")
            }

            val totalFrames = output.toIntOrNull()
            if (totalFrames == null) {
                throw NumberFormatException("Invalid frame count: $output")
            }

            val frameRate = totalFrames / videoDuration
            logger.info("Frame rate: $frameRate fps")
            return frameRate
        } catch (e: Exception) {
            logger.log(Level.SEVERE, "Error getting frame rate", e)
            throw e
        }
    }

Hii! Good that your program is evolving :slight_smile:

I added a method based on video - Fetch frame count with ffmpeg - Stack Overflow :

import org.openrndr.application
import org.slf4j.LoggerFactory
import java.io.File
import kotlin.math.roundToInt


fun main() = application {
    program {
        val logger = LoggerFactory.getLogger("test")

        fun getVideoDuration(path: File): Double {
            logger.info("Getting video duration for file: ${path.absolutePath}")
            val pbGetVideoDuration = ProcessBuilder(
                "ffprobe",
                "-v", "error",
                "-show_entries", "format=duration",
                "-of", "default=noprint_wrappers=1:nokey=1",
                path.absolutePath
            )

            try {
                val process = pbGetVideoDuration.start()
                val output = String(process.inputStream.readAllBytes()).trim()
                val exitCode = process.waitFor()

                if (exitCode != 0) {
                    val errorOutput = String(process.errorStream.readAllBytes())
                    throw Exception("ffprobe process failed with exit code $exitCode. Error: $errorOutput")
                }

                val duration = output.toDoubleOrNull() ?: throw NumberFormatException("Invalid video duration: $output")

                logger.info("Video duration: $duration seconds")
                return duration
            } catch (e: Exception) {
                logger.error("Error getting video duration", e)
                throw e
            }
        }

        fun getFrameRate(path: File, videoDuration: Double): Double {
            logger.info("Getting frame rate for file: ${path.absolutePath}")
            val pbGetFrameRate = ProcessBuilder(
                "ffprobe",
                "-v", "error",
                "-select_streams", "v:0",
                "-count_packets",
                "-show_entries", "stream=nb_read_packets",
                "-of", "csv=p=0",
                path.absolutePath
            )

            try {
                val process = pbGetFrameRate.start()
                val output = String(process.inputStream.readAllBytes()).trim()
                val exitCode = process.waitFor()

                if (exitCode != 0) {
                    val errorOutput = String(process.errorStream.readAllBytes())
                    throw Exception("ffprobe process failed with exit code $exitCode. Error: $errorOutput")
                }

                val totalFrames = output.toIntOrNull() ?: throw NumberFormatException("Invalid frame count: $output")

                val frameRate = totalFrames / videoDuration
                logger.info("Frame rate: $frameRate fps")
                return frameRate
            } catch (e: Exception) {
                logger.error("Error getting frame rate", e)
                throw e
            }
        }

        fun getVideoFrameCount(path: File): Int {
            val pbGetVideoFrameCount = ProcessBuilder(
                "ffprobe",
                "-v", "error",
                "-select_streams", "v:0",
                "-count_packets",
                "-show_entries", "stream=nb_read_packets",
                "-of", "csv=p=0",
                path.absolutePath
            )

            val frameCount = String(pbGetVideoFrameCount.start().inputStream.readAllBytes()).trim()
            return frameCount.toInt()
        }

        val vid = File("/tmp/v.mp4")
        val videoFrameCount = getVideoFrameCount(vid)
        val videoDuration = getVideoDuration(vid)
        val videoFrameRate = getFrameRate(vid, videoDuration)
        println("Frames: $videoFrameCount")
        println("Duration: $videoDuration")
        println("Frame Rate: $videoFrameRate")
        println("Dur*FR: ${(videoDuration * videoFrameRate).roundToInt()}")
    }
}

which prints

INFO [main] test                            ↘ Getting video duration for file: /tmp/v.mp4
 INFO [main] test                            ↘ Video duration: 105.333333 seconds
 INFO [main] test                            ↘ Getting frame rate for file: /tmp/v.mp4
 INFO [main] test                            ↘ Frame rate: 30.00000009493671 fps
Frames: 3160
Duration: 105.333333
Frame Rate: 30.00000009493671
Dur*FR: 3160

It uses the faster alternative with packets, which should match the number of frames. This takes about a second. Getting the actual frames takes a long time, maybe 15 or 20 seconds for the video I tested with.

How did it fail for you? Your code (with minor tweaks to pass the duration to the second method) seem to work.

If getting the packets is not good enough it seems there’s a program called mediainfo which can return the frame count.

One thing that I wonder about: what happens with variable frame rate videos? Does it matter? How common is that? I guess such videos could be converted to constant frame rate.

ps. I also saw another alternative by piping all the frames to /dev/null and counting them, which takes a few seconds.

I guess I was too sleepy and stressed at the time I posted and I didn’t state clearly what I was going through, neither paste the relevant code, LOL.

In fact, these snippets I pasted work fine. I don’t know why I pasted only them, without the rest of the program.

What I’m struggling to do is use the VideoFrames method you posted above to extract all video frames, process them and re-compile them into a video again in a single pass.

Even with the correct frame count, duration and framerate, I can’t code a program that uses this info to make all this in one swoop.

I don’t know if I made myself clear :confused: English is not my first language.
The file I’m working on is quite long.

If I run it two separate times, I get the desired result. It first extracts all the frames and exports them. If I run it again, it then compiles these frames into a video.

But it doesn’t do this in a single run. I can’t get the program to know that we have already extracted 100% of the frames, despite the fact it has even a progress counter which shows when 100% is achieved.

I’m trying to achieve this behavior in this isolated program so I can later transfer it to my main video exporter.

I will try a new approach with the new code you came up with.

Here’s the program

package exporter

import org.openrndr.application
import org.openrndr.color.ColorRGBa
import org.openrndr.draw.*
import org.openrndr.extra.imageFit.FitMethod
import org.openrndr.extra.imageFit.imageFit
import org.openrndr.ffmpeg.VideoWriter
import org.openrndr.math.IntVector2
import java.io.File

fun getVideoDimensions(path: File): IntVector2 {
    val pbGetVideoDimensions = ProcessBuilder(
        "ffprobe",
        "-v", "error",
        "-select_streams", "v:0",
        "-show_entries", "stream=width,height",
        "-of", "csv=s=x:p=0",
        path.absolutePath
    )

    val videoDimensions = String(pbGetVideoDimensions.start().inputStream.readAllBytes()).trim().split("x")
    return IntVector2(
        videoDimensions[0].toInt(),
        videoDimensions[1].toInt()
    )
}

/**
 * A class to provide access to all the frames of a video file.
 * It extracts all frames of the [path] video file into a subfolder in PNG format.
 */
class AbeXport(path: File, private val drawer: Drawer) {
    private val videoDimensions = getVideoDimensions(path)
    private var rt = renderTarget(videoDimensions.x, videoDimensions.y) {
        colorBuffer()
    }

    // A folder with the same name as the video file with ".frames" appended
    private val folder = File(path.absolutePath + ".frames")

    // Don't throw random files into the generated folder, it will mess up
    // the frameCount calculation
    var frameCount = folder.listFiles()?.size ?: 0
        private set

    // Used to avoid regenerating a frame if it was just requested.
    // Useful if we draw the same frame multiple times (aka pause).
    private var requestedFrame = -1

    // Returns a frame loaded from disk, or TRANSPARENT if out of range
    operator fun get(frame: Int): ColorBuffer {
        if (frame != requestedFrame) {
            drawer.isolatedWithTarget(rt) {
                ortho(rt)
                if (frame < 0 || frame >= frameCount) {
                    clear(ColorRGBa.TRANSPARENT)
                } else {
                    val imagePath = "$folder/frame${String.format("%05d", frame + 1)}.png"
                    val loadedImage = loadImage(imagePath)
                    image(loadedImage)
                    loadedImage.destroy()
                }
            }
            requestedFrame = frame
        }
        return rt.colorBuffer(0)
    }

    // Create the frame folder if it doesn't exist
    // and fill it with extracted frames
    init {
        if (!folder.exists()) {
            folder.mkdir()
        }
        if (frameCount == 0) {
            val args = listOf(
                "ffmpeg",
                "-r", "1",
                "-i", path.absolutePath,
                "-r", "1",
                "$folder/frame%05d.png"
            )
            val pb = ProcessBuilder(args)
            pb.start().waitFor()
            frameCount = folder.listFiles()?.size ?: 0
            println("$frameCount frames extracted")
        } else {
            println("$frameCount frames found")
        }
    }

    fun destroy() = rt.destroy()
}

// A sample program making use of VideoFrames.
// It just draws all the frames from in.mp4 into out.mp4, adding a text on top.
fun main() = application {
    program {
        val videoWriter = VideoWriter()
        videoWriter.size(width, height)
        videoWriter.output("C:/videos/godgangs/playlist/01-out.mp4")
        videoWriter.start()

        val canvas = renderTarget(width, height) {
            colorBuffer()
            depthBuffer()
        }

        val frames = AbeXport(File("C:/videos/godgangs/playlist/01.mp4"), drawer)

        val font = loadFont("data/fonts/default.otf", 200.0)

        for (i in 0 until frames.frameCount) {
            val frame = frames[i]
            drawer.isolatedWithTarget(canvas) {
                imageFit(frame, bounds, fitMethod = FitMethod.Cover)
                fill = ColorRGBa.CYAN
                fontMap = font
                text("$i", 10.0, height - 20.0)
            }
            // Save the canvas into the video file
            videoWriter.frame(canvas.colorBuffer(0))
            println("Saved frame $i")
        }
        videoWriter.stop()
        frames.destroy()

        println("Done")

        extend {
        }
    }
}

answering your questions:

i don’t know about variable frame rate videos. didn’t tackle this matter yet. but it is something i want to do when I get this right. I haven’t seen one yet so I don’t really know.

I am thinking about something with fixed framerate videos that do not match my program’s framerate (30fps). when handling a 60fps video, I’ll try to make the program detect this and render every other frame instead of 100% of the frames so the times match.

For 24fps videos, I’ll try adding some frame interpolation.

Might as well get somewhere where the program can calculate how to render these frames automatically.

But first things first, I guess :slight_smile:

I guess my problem is this: I like the VideoFrames method you posted here before because it’s so fast. It extracts all frames very quickly and also compiles them very fast, which I want.

I think it’s downside is that there is no way to know that 100% of the frames have been extracted and move on with the video writing process.

I got it done!!!

package exporter

import org.openrndr.application
import org.openrndr.color.ColorRGBa
import org.openrndr.draw.*
import org.openrndr.ffmpeg.VideoWriter
import org.openrndr.math.IntVector2
import org.slf4j.LoggerFactory
import java.io.File
import kotlin.math.roundToInt
import kotlinx.coroutines.*

fun main() = application {
    program {
        val logger = LoggerFactory.getLogger("VideoProcessor")

        fun getVideoDimensions(path: File): IntVector2 {
            val pbGetVideoDimensions = ProcessBuilder(
                "ffprobe",
                "-v", "error",
                "-select_streams", "v:0",
                "-show_entries", "stream=width,height",
                "-of", "csv=s=x:p=0",
                path.absolutePath
            )

            val videoDimensions = String(pbGetVideoDimensions.start().inputStream.readAllBytes()).trim().split("x")
            return IntVector2(
                videoDimensions[0].toInt(),
                videoDimensions[1].toInt()
            )
        }

        fun getVideoDuration(path: File): Double {
            logger.info("Getting video duration for file: ${path.absolutePath}")
            val pbGetVideoDuration = ProcessBuilder(
                "ffprobe",
                "-v", "error",
                "-show_entries", "format=duration",
                "-of", "default=noprint_wrappers=1:nokey=1",
                path.absolutePath
            )

            try {
                val process = pbGetVideoDuration.start()
                val output = String(process.inputStream.readAllBytes()).trim()
                val exitCode = process.waitFor()

                if (exitCode != 0) {
                    val errorOutput = String(process.errorStream.readAllBytes())
                    throw Exception("ffprobe process failed with exit code $exitCode. Error: $errorOutput")
                }

                val duration = output.toDoubleOrNull() ?: throw NumberFormatException("Invalid video duration: $output")

                logger.info("Video duration: $duration seconds")
                return duration
            } catch (e: Exception) {
                logger.error("Error getting video duration", e)
                throw e
            }
        }

        fun getFrameRate(path: File): Double {
            logger.info("Getting frame rate for file: ${path.absolutePath}")
            val pbGetFrameRate = ProcessBuilder(
                "ffprobe",
                "-v", "error",
                "-select_streams", "v:0",
                "-show_entries", "stream=r_frame_rate",
                "-of", "default=noprint_wrappers=1:nokey=1",
                path.absolutePath
            )

            try {
                val process = pbGetFrameRate.start()
                val output = String(process.inputStream.readAllBytes()).trim()
                val exitCode = process.waitFor()

                if (exitCode != 0) {
                    val errorOutput = String(process.errorStream.readAllBytes())
                    throw Exception("ffprobe process failed with exit code $exitCode. Error: $errorOutput")
                }

                val (numerator, denominator) = output.split("/").map { it.toDouble() }
                val frameRate = numerator / denominator

                logger.info("Frame rate: $frameRate fps")
                return frameRate
            } catch (e: Exception) {
                logger.error("Error getting frame rate", e)
                throw e
            }
        }

        class VideoProcessor(private val inputPath: File, private val outputPath: File, private val drawer: Drawer) {
            private val videoDimensions = getVideoDimensions(inputPath)
            private val videoDuration = getVideoDuration(inputPath)
            private val frameRate = getFrameRate(inputPath)
            private val totalFrames = (videoDuration * frameRate).roundToInt()

            private val rt = renderTarget(videoDimensions.x, videoDimensions.y) {
                colorBuffer()
            }

            private val folder = File(inputPath.absolutePath + ".frames")
            private lateinit var videoWriter: VideoWriter

            init {
                if (!folder.exists()) {
                    folder.mkdir()
                }
            }

            private suspend fun extractFrames() {
                println("Extracting frames...")
                val extractArgs = listOf(
                    "ffmpeg",
                    "-i", inputPath.absolutePath,
                    "-vf", "fps=${frameRate.roundToInt()}",
                    "$folder/frame%05d.png"
                )
                val pbExtract = ProcessBuilder(extractArgs)
                val process = pbExtract.start()

                // Start a coroutine to check the frame count
                coroutineScope {
                    launch {
                        var lastCheckedCount = 0
                        while (isActive) {
                            delay(1000) // Check every second
                            val currentCount = folder.listFiles()?.size ?: 0
                            if (currentCount > lastCheckedCount) {
                                println("Extracted $currentCount frames out of $totalFrames")
                                lastCheckedCount = currentCount
                            }
                            if (currentCount >= totalFrames) {
                                println("All frames extracted")
                                process.destroy()
                                break
                            }
                        }
                    }
                }

                process.waitFor()
            }

            private fun processFrames() {
                println("Processing frames and compiling video...")
                videoWriter = VideoWriter().apply {
                    size(videoDimensions.x, videoDimensions.y)
                    output(outputPath.absolutePath)
                    frameRate = this@VideoProcessor.frameRate.roundToInt()
                    start()
                }

                val font = loadFont("data/fonts/default.otf", 200.0)

                for (i in 1..totalFrames) {
                    val framePath = "$folder/frame${String.format("%05d", i)}.png"
                    val frameFile = File(framePath)
                    if (!frameFile.exists()) {
                        println("Frame $i not found, stopping processing")
                        break
                    }

                    val frame = loadImage(framePath)

                    drawer.isolatedWithTarget(rt) {
                        ortho(rt)
                        image(frame)
                        fill = ColorRGBa.CYAN
                        fontMap = font
                        text("${i-1}", 10.0, videoDimensions.y - 20.0)
                    }

                    videoWriter.frame(rt.colorBuffer(0))
                    frame.destroy()
                    println("Processed frame ${i-1}")
                }

                videoWriter.stop()
            }

            suspend fun processVideo() {
                extractFrames()
                processFrames()
                rt.destroy()
                folder.deleteRecursively()
                println("Video processing completed")
            }
        }

        runBlocking {
            val inputVideo = File("C:/videos/godgangs/playlist/01.mp4")
            val outputVideo = File("C:/videos/godgangs/playlist/01-out.mp4")
            val processor = VideoProcessor(inputVideo, outputVideo, drawer)
            processor.processVideo()
        }

        extend {
            // This is left empty as we don't need to continuously update the window
        }
    }
}

It extracts all the frames from the video and, while it does it, compares the number of files inside the extract folder to the total frame count (AI wrote the code but this was my idea :wink: )

When the number matches, it quits extracting frames and proceeds to write the video.
This is very fast and a very solid foundation for what I want to achieve.

Thanks again for your help.

1 Like

Yay! Very happy to read you made it :slight_smile:

I’m thinking of writing a method that parses the output of

ffprobe -v quiet -print_format json -show_format -show_streams fast2.mp4

so one doesn’t need to execute programs multiple times.

The output seems to have everything there:

{
    "streams": [
        {
            "index": 0,
            "codec_name": "h264",
            "codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10",
            "profile": "High",
            "codec_type": "video",
            "codec_tag_string": "avc1",
            "codec_tag": "0x31637661",
            "width": 1080,
            "height": 1920,
            "coded_width": 1080,
            "coded_height": 1920,
            "closed_captions": 0,
            "film_grain": 0,
            "has_b_frames": 2,
            "sample_aspect_ratio": "1:1",
            "display_aspect_ratio": "9:16",
            "pix_fmt": "yuvj420p",
            "level": 40,
            "color_range": "pc",
            "color_space": "bt709",
            "color_transfer": "bt709",
            "color_primaries": "bt709",
            "chroma_location": "left",
            "field_order": "progressive",
            "refs": 1,
            "is_avc": "true",
            "nal_length_size": "4",
            "id": "0x1",
            "r_frame_rate": "30/1",
            "avg_frame_rate": "30/1",
            "time_base": "1/15360",
            "start_pts": 0,
            "start_time": "0.000000",
            "duration_ts": 1617920,
            "duration": "105.333333",
            "bit_rate": "4361487",
            "bits_per_raw_sample": "8",
            "nb_frames": "3160",
            "extradata_size": 51,
            "disposition": {
                "default": 1,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0,
                "non_diegetic": 0,
                "captions": 0,
                "descriptions": 0,
                "metadata": 0,
                "dependent": 0,
                "still_image": 0
            },
            "tags": {
                "language": "und",
                "handler_name": "VideoHandler",
                "vendor_id": "[0][0][0][0]",
                "encoder": "Lavc60.31.102 libx264"
            }
        },
        {
            "index": 1,
            "codec_name": "aac",
            "codec_long_name": "AAC (Advanced Audio Coding)",
            "profile": "LC",
            "codec_type": "audio",
            "codec_tag_string": "mp4a",
            "codec_tag": "0x6134706d",
            "sample_fmt": "fltp",
            "sample_rate": "48000",
            "channels": 2,
            "channel_layout": "stereo",
            "bits_per_sample": 0,
            "initial_padding": 0,
            "id": "0x2",
            "r_frame_rate": "0/0",
            "avg_frame_rate": "0/0",
            "time_base": "1/48000",
            "start_pts": 0,
            "start_time": "0.000000",
            "duration_ts": 5047008,
            "duration": "105.146000",
            "bit_rate": "128502",
            "nb_frames": "4930",
            "extradata_size": 5,
            "disposition": {
                "default": 1,
                "dub": 0,
                "original": 0,
                "comment": 0,
                "lyrics": 0,
                "karaoke": 0,
                "forced": 0,
                "hearing_impaired": 0,
                "visual_impaired": 0,
                "clean_effects": 0,
                "attached_pic": 0,
                "timed_thumbnails": 0,
                "non_diegetic": 0,
                "captions": 0,
                "descriptions": 0,
                "metadata": 0,
                "dependent": 0,
                "still_image": 0
            },
            "tags": {
                "language": "und",
                "handler_name": "SoundHandler",
                "vendor_id": "[0][0][0][0]"
            }
        }
    ],
    "format": {
        "filename": "fast2.mp4",
        "nb_streams": 2,
        "nb_programs": 0,
        "nb_stream_groups": 0,
        "format_name": "mov,mp4,m4a,3gp,3g2,mj2",
        "format_long_name": "QuickTime / MOV",
        "start_time": "0.000000",
        "duration": "105.333333",
        "size": "59226768",
        "bit_rate": "4498235",
        "probe_score": 100,
        "tags": {
            "major_brand": "isom",
            "minor_version": "512",
            "compatible_brands": "isomiso2avc1mp41",
            "encoder": "Lavf60.16.100"
        }
    }
}

Update : I have now a working prototype.