From f9a972acaab55144a56bbc5c1daa1dc28e8f4aef Mon Sep 17 00:00:00 2001 From: Grayson Earle Date: Fri, 21 May 2021 18:15:46 +0200 Subject: [PATCH] added an example for StyleGAN2 inference --- example-StyleGAN2/Makefile | 13 +++ example-StyleGAN2/addons.make | 6 ++ example-StyleGAN2/config.make | 142 ++++++++++++++++++++++++++++++++ example-StyleGAN2/src/main.cpp | 14 ++++ example-StyleGAN2/src/ofApp.cpp | 118 ++++++++++++++++++++++++++ example-StyleGAN2/src/ofApp.h | 32 +++++++ 6 files changed, 325 insertions(+) create mode 100644 example-StyleGAN2/Makefile create mode 100644 example-StyleGAN2/addons.make create mode 100644 example-StyleGAN2/config.make create mode 100644 example-StyleGAN2/src/main.cpp create mode 100644 example-StyleGAN2/src/ofApp.cpp create mode 100644 example-StyleGAN2/src/ofApp.h diff --git a/example-StyleGAN2/Makefile b/example-StyleGAN2/Makefile new file mode 100644 index 0000000..8d8e4c0 --- /dev/null +++ b/example-StyleGAN2/Makefile @@ -0,0 +1,13 @@ +# Attempt to load a config.make file. +# If none is found, project defaults in config.project.make will be used. +ifneq ($(wildcard config.make),) + include config.make +endif + +# make sure the the OF_ROOT location is defined +ifndef OF_ROOT + OF_ROOT=$(realpath ../../..) +endif + +# call the project makefile! +include $(OF_ROOT)/libs/openFrameworksCompiled/project/makefileCommon/compile.project.mk diff --git a/example-StyleGAN2/addons.make b/example-StyleGAN2/addons.make new file mode 100644 index 0000000..61a936f --- /dev/null +++ b/example-StyleGAN2/addons.make @@ -0,0 +1,6 @@ +ofxHTTP +ofxIO +ofxMediaType +ofxNetworkUtils +ofxSSLManager +ofxRunway diff --git a/example-StyleGAN2/config.make b/example-StyleGAN2/config.make new file mode 100644 index 0000000..e01ac7b --- /dev/null +++ b/example-StyleGAN2/config.make @@ -0,0 +1,142 @@ +################################################################################ +# CONFIGURE PROJECT MAKEFILE (optional) +# This file is where we make project specific configurations. +################################################################################ + +################################################################################ +# OF ROOT +# The location of your root openFrameworks installation +# (default) OF_ROOT = ../../../../../.. +################################################################################ +# OF_ROOT = ../../../../../.. + +################################################################################ +# PROJECT ROOT +# The location of the project - a starting place for searching for files +# (default) PROJECT_ROOT = . (this directory) +# +################################################################################ +# PROJECT_ROOT = . + +################################################################################ +# PROJECT SPECIFIC CHECKS +# This is a project defined section to create internal makefile flags to +# conditionally enable or disable the addition of various features within +# this makefile. For instance, if you want to make changes based on whether +# GTK is installed, one might test that here and create a variable to check. +################################################################################ +# None + +################################################################################ +# PROJECT EXTERNAL SOURCE PATHS +# These are fully qualified paths that are not within the PROJECT_ROOT folder. +# Like source folders in the PROJECT_ROOT, these paths are subject to +# exlclusion via the PROJECT_EXLCUSIONS list. +# +# (default) PROJECT_EXTERNAL_SOURCE_PATHS = (blank) +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_EXTERNAL_SOURCE_PATHS = + +################################################################################ +# PROJECT EXCLUSIONS +# These makefiles assume that all folders in your current project directory +# and any listed in the PROJECT_EXTERNAL_SOURCH_PATHS are are valid locations +# to look for source code. The any folders or files that match any of the +# items in the PROJECT_EXCLUSIONS list below will be ignored. +# +# Each item in the PROJECT_EXCLUSIONS list will be treated as a complete +# string unless teh user adds a wildcard (%) operator to match subdirectories. +# GNU make only allows one wildcard for matching. The second wildcard (%) is +# treated literally. +# +# (default) PROJECT_EXCLUSIONS = (blank) +# +# Will automatically exclude the following: +# +# $(PROJECT_ROOT)/bin% +# $(PROJECT_ROOT)/obj% +# $(PROJECT_ROOT)/%.xcodeproj +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_EXCLUSIONS = + +################################################################################ +# PROJECT LINKER FLAGS +# These flags will be sent to the linker when compiling the executable. +# +# (default) PROJECT_LDFLAGS = -Wl,-rpath=./libs +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ + +# Currently, shared libraries that are needed are copied to the +# $(PROJECT_ROOT)/bin/libs directory. The following LDFLAGS tell the linker to +# add a runtime path to search for those shared libraries, since they aren't +# incorporated directly into the final executable application binary. +# TODO: should this be a default setting? +# PROJECT_LDFLAGS=-Wl,-rpath=./libs + +################################################################################ +# PROJECT DEFINES +# Create a space-delimited list of DEFINES. The list will be converted into +# CFLAGS with the "-D" flag later in the makefile. +# +# (default) PROJECT_DEFINES = (blank) +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_DEFINES = + +################################################################################ +# PROJECT CFLAGS +# This is a list of fully qualified CFLAGS required when compiling for this +# project. These CFLAGS will be used IN ADDITION TO the PLATFORM_CFLAGS +# defined in your platform specific core configuration files. These flags are +# presented to the compiler BEFORE the PROJECT_OPTIMIZATION_CFLAGS below. +# +# (default) PROJECT_CFLAGS = (blank) +# +# Note: Before adding PROJECT_CFLAGS, note that the PLATFORM_CFLAGS defined in +# your platform specific configuration file will be applied by default and +# further flags here may not be needed. +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_CFLAGS = + +################################################################################ +# PROJECT OPTIMIZATION CFLAGS +# These are lists of CFLAGS that are target-specific. While any flags could +# be conditionally added, they are usually limited to optimization flags. +# These flags are added BEFORE the PROJECT_CFLAGS. +# +# PROJECT_OPTIMIZATION_CFLAGS_RELEASE flags are only applied to RELEASE targets. +# +# (default) PROJECT_OPTIMIZATION_CFLAGS_RELEASE = (blank) +# +# PROJECT_OPTIMIZATION_CFLAGS_DEBUG flags are only applied to DEBUG targets. +# +# (default) PROJECT_OPTIMIZATION_CFLAGS_DEBUG = (blank) +# +# Note: Before adding PROJECT_OPTIMIZATION_CFLAGS, please note that the +# PLATFORM_OPTIMIZATION_CFLAGS defined in your platform specific configuration +# file will be applied by default and further optimization flags here may not +# be needed. +# +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_OPTIMIZATION_CFLAGS_RELEASE = +# PROJECT_OPTIMIZATION_CFLAGS_DEBUG = + +################################################################################ +# PROJECT COMPILERS +# Custom compilers can be set for CC and CXX +# (default) PROJECT_CXX = (blank) +# (default) PROJECT_CC = (blank) +# Note: Leave a leading space when adding list items with the += operator +################################################################################ +# PROJECT_CXX = +# PROJECT_CC = diff --git a/example-StyleGAN2/src/main.cpp b/example-StyleGAN2/src/main.cpp new file mode 100644 index 0000000..8bce185 --- /dev/null +++ b/example-StyleGAN2/src/main.cpp @@ -0,0 +1,14 @@ +#include "ofMain.h" +#include "ofApp.h" + +//======================================================================== +int main( ){ + + ofSetupOpenGL(1024,768, OF_WINDOW); // <-------- setup the GL context + + // this kicks off the running of my app + // can be OF_WINDOW or OF_FULLSCREEN + // pass in width and height too: + ofRunApp( new ofApp()); + +} diff --git a/example-StyleGAN2/src/ofApp.cpp b/example-StyleGAN2/src/ofApp.cpp new file mode 100644 index 0000000..8f6dc60 --- /dev/null +++ b/example-StyleGAN2/src/ofApp.cpp @@ -0,0 +1,118 @@ +#include "ofApp.h" + +/* + + an example of GAN inference (generating an image from a GAN) + this is setup for a network that wants an input vector z of 512 values + and a truncation value, but should work with others if you modify it + + try using StyleGAN2 with RunwayML to generate some faces + this example assumes Local GPU + +*/ + +//-------------------------------------------------------------- + +void ofApp::setup() +{ + // uncomment the following line if you want a verbose log (which means a lot of info will be printed) + // ofSetLogLevel(OF_LOG_VERBOSE); + + ofSetWindowShape(512, 512); + + // setup Runway + runway.setup(this, "http://localhost:8000"); + runway.start(); + + // infer a new image to begin + generate_image(generate_random_z(), truncation); +} +//-------------------------------------------------------------- +void ofApp::update() +{ + + // infer/generate a new image if we have sent a request to Runway + if (bWaitingForResponse) + { + runway.get("image", currentImg); // stores generated image to currentImg + bWaitingForResponse = false; + } + +} +//-------------------------------------------------------------- +void ofApp::draw() +{ + + // draw image received from Runway + if (currentImg.isAllocated()) + { + currentImg.draw(0, 0); + } + + // draw Runway's status. It returns the bounding box of the drawn text. It is useful so you can draw other stuff and avoid overlays + ofRectangle r = runway.drawStatus(620, 440, true); + ofDrawBitmapString("Press ' ' to send to Runway", r.getBottomLeft() + glm::vec3(0, 20, 0)); + + + // save frame (useful for latent space walks) + // string fn = ofToString(ofGetFrameNum(),4,'0')+".png"; + // cout << fn << endl; + // ofSaveScreen(fn); +} + +//-------------------------------------------------------------- +void ofApp::keyReleased(int key) +{ + // on spacebar, infer a new image + if (key == ' ') + { + generate_image(generate_random_z(), truncation); + } +} + +//-------------------------------------------------------------- +vector ofApp::generate_random_z() +{ + // this is a helpful method to generate a random set of input values + + vector z; // create an empty vector of floats + for (int i = 0; i < 512; i++) + { + z.push_back(ofRandom(-min_max_vals, min_max_vals)); + } + + return z; +} + +//-------------------------------------------------------------- +void ofApp::generate_image(vector z, float truncation) +{ + // skip if content image isn't loaded yet + + if (runway.isBusy()) + return; + + // create a data object to send + ofxRunwayData data; + data.setFloats("z", z); // add our vector of random floats + data.setFloat("truncation", truncation); // also need to set a truncation + + // basically, the higher the truncation value the weirder the results + + runway.send(data); + + bWaitingForResponse = true; +} + +// Runway sends information about the current model +//-------------------------------------------------------------- +void ofApp::runwayInfoEvent(ofJson &info) +{ + ofLogNotice("ofApp::runwayInfoEvent") << info.dump(2); +} +// if anything goes wrong +//-------------------------------------------------------------- +void ofApp::runwayErrorEvent(string &message) +{ + ofLogNotice("ofApp::runwayErrorEvent") << message; +} diff --git a/example-StyleGAN2/src/ofApp.h b/example-StyleGAN2/src/ofApp.h new file mode 100644 index 0000000..085f481 --- /dev/null +++ b/example-StyleGAN2/src/ofApp.h @@ -0,0 +1,32 @@ +#pragma once + +#include "ofMain.h" +#include "ofxRunway.h" + +// make sure you make your ofApp or class to inherit from ofxRunwayListener +class ofApp : public ofBaseApp, public ofxRunwayListener{ + +public: + void setup(); + void update(); + void draw(); + void keyReleased(int key); + + ofxRunway runway; + + ofImage currentImg; + + vector generate_random_z(); + void generate_image(vector z, float truncation); + + // Callback functions that process what Runway sends back + void runwayInfoEvent(ofJson& info); + void runwayErrorEvent(string& message); + + bool bWaitingForResponse = false; + + // GAN options + float truncation = 1.5; + float min_max_vals = 2; + +};