basic ebsynth support

scripts/Berry_Method.py

@@ -29,8 +29,14 @@
 diffuse = False
 check_edges = False
 
-def split_into_batches(frames, batch_size):
-    return [frames[i:i + batch_size] for i in range(0, len(frames), batch_size)]
+def split_into_batches(frames, batch_size, max_batches):
+    groups = [frames[i:i+batch_size] for i in range(0, len(frames), batch_size)][:max_batches]
+
+    # Add any remaining images to the last group
+    if len(frames) > max_batches * batch_size:
+        groups[-1] += frames[max_batches*batch_size:]
+
+    return groups
 
 def create_square_texture(frames, max_size, side_length=3):
     num_frames = len(frames)
@@ -55,7 +61,7 @@ def create_square_texture(frames, max_size, side_length=3):
 
     return texture
 
-def split_frames_into_big_batches(frames, batch_size, border):
+def split_frames_into_big_batches(frames, batch_size, border,ebsynth,returnframe_locations=False):
     """
     Splits an array of numpy frames into batches of a given size, adding a certain number of border
     frames from the next batch to each batch.
@@ -72,28 +78,39 @@ def split_frames_into_big_batches(frames, batch_size, border):
     num_batches = int(np.ceil(num_frames / batch_size))
     print(f"frames num = {len(frames)} while num batches = {num_batches}")
     batches = []
+
+    frame_locations = []
     for i in range(num_batches):
         start_idx = i * batch_size
         end_idx = start_idx + batch_size
-
-        # Add border frames if not the last batch and if available
-        if i < num_batches - 1:
-            end_idx += min(border, num_frames - end_idx)
+        if ebsynth == False:
+            # Add border frames if not the last batch and if available
+            if i < num_batches - 1:
+                end_idx += min(border, num_frames - end_idx)
+            else:
+                # Combine the last batch with the previous batch if the number of frames in the last batch is smaller than the border size
+                if end_idx - start_idx < border and len(batches) > 0:
+                    batches[-1] = np.concatenate((batches[-1], frames[start_idx:end_idx]))
+                    break
         else:
-            # Combine the last batch with the previous batch if the number of frames in the last batch is smaller than the border size
-            if end_idx - start_idx < border and len(batches) > 0:
-                batches[-1] = np.concatenate((batches[-1], frames[start_idx:end_idx]))
-                break
+            if i < num_batches - 1:
+                end_idx = end_idx + border
 
         end_idx = min(end_idx, num_frames)
         batches.append(frames[start_idx:end_idx])
-    return batches
+        frame_locations.append((start_idx,end_idx))
 
-def split_square_texture(texture, num_frames, _smol_resolution):
+    if returnframe_locations == False:
+        return batches
+    else:
+        return batches,frame_locations
+
+def split_square_texture(texture, num_frames,max_frames, _smol_resolution):
+
     texture_height, texture_width = texture.shape[:2]
     texture_aspect_ratio = float(texture_width) / float(texture_height)
 
-    frames_per_row = int(math.ceil(math.sqrt(num_frames)))
+    frames_per_row = int(math.ceil(math.sqrt(max_frames)))
     frame_height = texture_height // frames_per_row
     frame_width = int(frame_height * texture_aspect_ratio)
 
@@ -138,7 +155,8 @@ def generate_square_from_video(video_path, fps, batch_size,resolution,size_size)
     frames_limit = (size_size * size_size) * batch_size
     frames = extract_frames_movpie(video_data, fps, frames_limit)
     print(len(frames))
-    batches = split_into_batches(frames, batch_size)
+    number_of_batches = size_size * size_size
+    batches = split_into_batches(frames, batch_size,number_of_batches)
     print("Number of batches:", len(batches))
     first_frames = [batch[0] for batch in batches]  
 
@@ -147,11 +165,13 @@ def generate_square_from_video(video_path, fps, batch_size,resolution,size_size)
 
     return square_texture
 
-def generate_squares_to_folder (video_path, fps, batch_size,resolution,size_size,max_frames,output_folder,border):
+def generate_squares_to_folder (video_path, fps, batch_size,resolution,size_size,max_frames,output_folder,border,ebsynth_mode):
+
+    if ebsynth_mode == False:
+        if border >=  (batch_size * size_size * size_size) / 2:
+            raise Exception("too many border frames, reduce border or increase batch size")
+
 
-    if border >=  (batch_size * size_size * size_size) / 2:
-        raise Exception("too many border frames, reduce border or increase batch size")
-
     input_folder_loc = os.path.join(output_folder, "input")
     output_folder_loc = os.path.join(output_folder, "output")
     debug_result = os.path.join(output_folder, "result")
@@ -163,17 +183,33 @@ def generate_squares_to_folder (video_path, fps, batch_size,resolution,size_size
         os.makedirs(output_folder_loc)
     if not os.path.exists(debug_result):
         os.makedirs(debug_result)
+    frames_loc = os.path.join(output_folder, "frames")
+    keys_loc = os.path.join(output_folder, "keys")
+
+    if ebsynth_mode == True:
+        if not os.path.exists(frames_loc):
+            os.makedirs(frames_loc)
+        if not os.path.exists(keys_loc):
+            os.makedirs(keys_loc)
+
     video_data = convert_video_to_bytes(video_path)
-    per_batch_limmit = ((size_size * size_size) * batch_size) - border
+    per_batch_limmit = ((size_size * size_size) * batch_size) + border
+    #if ebsynth_mode == False:
+    #    per_batch_limmit = per_batch_limmit + border
     frames = extract_frames_movpie(video_data, fps, max_frames)
-    bigbatches = split_frames_into_big_batches(frames, per_batch_limmit,border)
+
+
+    bigbatches = split_frames_into_big_batches(frames, per_batch_limmit,border,ebsynth=ebsynth_mode)
     square_textures = []
     for i in range(len(bigbatches)):
-        batches = split_into_batches(bigbatches[i], batch_size)
+        batches = split_into_batches(bigbatches[i], batch_size, size_size * size_size)
         print("Number of batches:", len(batches))
-        first_frames = [batch[0] for batch in batches]  
+        if ebsynth_mode == False:
+            keyframes = [batch[0] for batch in batches]  
+        else: 
+            keyframes = [batch[int(len(batch)/2)] for batch in batches]
 
-        square_texture = create_square_texture(first_frames, resolution,side_length=size_size)
+        square_texture = create_square_texture(keyframes, resolution,side_length=size_size)
         save_square_texture(square_texture, os.path.join(input_folder_loc, f"input{i}.png"))
         square_textures.append(square_texture)
 
@@ -189,8 +225,6 @@ def generate_squares_to_folder (video_path, fps, batch_size,resolution,size_size
     return square_texture
 
 
-
-
 def delete_folder_contents(folder_path):
     for filename in os.listdir(folder_path):
         file_path = os.path.join(folder_path, filename)
@@ -245,11 +279,11 @@ def process_video_batch (video_path, fps, per_side, batch_size, fillindenoise, e
     video_data = convert_video_to_bytes(video_path)
     frames = extract_frames_movpie(video_data, fps, max_frames)
     print(f"full frames num = {len(frames)}")
-    bigbatches = split_frames_into_big_batches(frames, per_batch_limmit,border)
+    bigbatches = split_frames_into_big_batches(frames, per_batch_limmit,border,ebsynth=False)
     bigprocessedbatches = []
     for i , batch in enumerate(bigbatches):
         if i < len(square_textures):
-            new_batch = process_video(batch, fps, batch_size, fillindenoise, edgedenoise, _smol_resolution,square_textures[i])
+            new_batch = process_video(batch, per_side, batch_size, fillindenoise, edgedenoise, _smol_resolution,square_textures[i])
             bigprocessedbatches.append(new_batch)
             for a, image in enumerate(new_batch):
                 Image.fromarray(image).save(os.path.join(output_folder, f"result/output{a + (len(new_batch) * i)}.png"))
@@ -282,30 +316,30 @@ def process_video_single(video_path, fps, per_side, batch_size, fillindenoise, e
     #rerun the generatesquarefromvideo function to get the unaltered square texture
     video_data = convert_video_to_bytes(video_path)
     frames = extract_frames_movpie(video_data, fps, frames_limit)
-    processed_frames = process_video(frames,fps,batch_size,fillindenoise,edgedenoise,_smol_resolution,square_texture)
+    processed_frames = process_video(frames,per_side,batch_size,fillindenoise,edgedenoise,_smol_resolution,square_texture)
     output_video_path = os.path.join(output_folder, "output.mp4")
     generated_video =  pil_images_to_video(processed_frames, output_video_path, fps)
     return generated_video
 
 
-def process_video(frames, fps, batch_size, fillindenoise, edgedenoise, _smol_resolution,square_texture):
+def process_video(frames, per_side, batch_size, fillindenoise, edgedenoise, _smol_resolution,square_texture):
 
     frame_count = 0
     print(len(frames))
-    batches = split_into_batches(frames, batch_size)
+    batches = split_into_batches(frames, batch_size, per_side * per_side)
     print("Number of batches:", len(batches))
     first_frames = [batch[0] for batch in batches]
     #actuallyprocessthevideo
     debug = False
-    encoded_square_texture = utilityb.texture_to_base64(square_texture)
+
     frame_count = 0
     global resolution
     print(len(frames))
 
 
     if debug is False:
         #result_texture = sdprocess.square_Image_request(encoded_square_texture, prompt, initial_denoise, resolution, seed)
-        result_texture = encoded_square_texture
+        result_texture = square_texture
         #save_square_texture(encoded_returns, "./result/processed.png")
 
     else:
@@ -315,9 +349,11 @@ def process_video(frames, fps, batch_size, fillindenoise, edgedenoise, _smol_res
         resolution_get = Image.open("./result/processed.png")
         resolution= resolution_get.height
         # this is stupid and inefficiant i dont care
-    encoded_returns = cv2.cvtColor(utilityb.base64_to_texture(result_texture), cv2.COLOR_BGR2RGB)
+    #its not really encoded anymore is it
+    encoded_returns = result_texture
+    #encoded_returns = cv2.cvtColor(utilityb.base64_to_texture(result_texture), cv2.COLOR_BGR2RGB)
 
-    new_frames = split_square_texture(encoded_returns, len(first_frames),_smol_resolution)
+    new_frames = split_square_texture(encoded_returns,len(first_frames), per_side * per_side,_smol_resolution)
 
     if check_edges:
         for i, image in enumerate(new_frames):

scripts/Ebsynth_Processing.py

@@ -0,0 +1,176 @@
+import os
+import glob
+#om nom nom nom
+import requests
+import json
+from pprint import pprint
+import base64
+import numpy as np
+from io import BytesIO
+import scripts.optical_flow_simple as opflow
+from PIL import Image, ImageOps,ImageFilter
+import io
+from collections import deque
+import cv2
+import scripts.Berry_Method as bmethod
+import scripts.berry_utility as butility
+import re
+
+
+
+def sort_into_folders(video_path, fps, per_side, batch_size, fillindenoise, edgedenoise, _smol_resolution,square_textures,max_frames,output_folder,border):
+    per_batch_limmit = (((per_side * per_side) * batch_size)) + border
+    video_data = bmethod.convert_video_to_bytes(video_path)
+    frames = bmethod.extract_frames_movpie(video_data, fps, max_frames)
+    print(f"full frames num = {len(frames)}")
+
+
+    output_frames_folder = os.path.join(output_folder, "frames")
+    output_keys_folder = os.path.join(output_folder, "keys")
+    for i, frame in enumerate(frames):
+        bmethod.save_square_texture(frame, os.path.join(output_frames_folder, "frames{:05d}.png".format(i)))
+    original_frame_height,original_frame_width = frames[0].shape[:2]
+    print (original_frame_height,original_frame_width)
+    bigbatches,frameLocs = bmethod.split_frames_into_big_batches(frames, per_batch_limmit,border,ebsynth=True,returnframe_locations=True)
+    bigprocessedbatches = []
+
+
+    print (len(square_textures))
+    for a,bigbatch in enumerate(bigbatches):
+        batches = bmethod.split_into_batches(bigbatches[a], batch_size,per_side* per_side)
+
+        keyframes = [batch[int(len(batch)/2)] for batch in batches]
+        if a < len(square_textures):
+            new_frames = bmethod.split_square_texture(square_textures[a],len(keyframes), per_side* per_side,_smol_resolution)
+            new_frame_start,new_frame_end = frameLocs[a]
+            for b in range(len(new_frames)):
+                print (new_frame_start)
+                inner_start = new_frame_start + (b * batch_size)
+                inner_end = new_frame_start + ((b+1) * batch_size)
+                frame_position  = inner_start + int((inner_end - inner_start)/2)
+                frame_to_save = cv2.resize(new_frames[b], (original_frame_width, original_frame_height), interpolation=cv2.INTER_LINEAR)
+                bmethod.save_square_texture(frame_to_save, os.path.join(output_keys_folder, "keys{:05d}.png".format(frame_position)))
+
+    just_frame_groups = []
+    for i in range(len(bigprocessedbatches)):
+        newgroup = []
+        for b in range(len(bigprocessedbatches[i])):
+            newgroup.append(bigprocessedbatches[i][b])
+        just_frame_groups.append(newgroup)
+
+    return
+
+
+def recombine (video_path, fps, per_side, batch_size, fillindenoise, edgedenoise, _smol_resolution,square_textures,max_frames,output_folder,border):
+    just_frame_groups = []
+    per_batch_limmit = (((per_side * per_side) * batch_size)) - border
+    video_data = bmethod.convert_video_to_bytes(video_path)
+    frames = bmethod.extract_frames_movpie(video_data, fps, max_frames)
+    bigbatches,frameLocs = bmethod.split_frames_into_big_batches(frames, per_batch_limmit,border,returnframe_locations=True)
+    bigprocessedbatches = []
+    for i in range(len(bigprocessedbatches)):
+        newgroup = []
+        for b in range(len(bigprocessedbatches[i])):
+            newgroup.append(bigprocessedbatches[i][b])
+        just_frame_groups.append(newgroup)
+
+    combined = bmethod.merge_image_batches(just_frame_groups, border)
+
+    save_loc = os.path.join(output_folder, "non_blended.mp4")
+    generated_vid = bmethod.pil_images_to_video(combined,save_loc, fps)
+
+
+
+def crossfade_images(image1, image2, alpha):
+    """Crossfade between two images with a given alpha value."""
+    image1 = image1.convert("RGBA")
+    image2 = image2.convert("RGBA")
+    return Image.blend(image1, image2, alpha)
+
+def crossfade_folder_of_folders(output_folder, fps):
+    """Crossfade between images in a folder of folders and save the results."""
+    root_folder = output_folder
+    all_dirs = [d for d in os.listdir(root_folder) if os.path.isdir(os.path.join(root_folder, d))]
+    dirs = [d for d in all_dirs if d.startswith("out_")]
+
+    dirs.sort()
+
+    output_images = []
+    allkeynums = getkeynums(os.path.join(root_folder, "keys"))
+    print(allkeynums)
+
+    for b in range(allkeynums[0]):
+        current_dir = os.path.join(root_folder, dirs[0])
+        images_current = sorted(os.listdir(current_dir))
+        image1_path = os.path.join(current_dir, images_current[b])
+        image1 = Image.open(image1_path)
+        output_images.append(np.array(image1))
+
+    for i in range(len(dirs) - 1):
+        current_dir = os.path.join(root_folder, dirs[i])
+        next_dir = os.path.join(root_folder, dirs[i + 1])
+
+        images_current = sorted(os.listdir(current_dir))
+        images_next = sorted(os.listdir(next_dir))
+
+        startnum = get_num_at_index(current_dir,0)
+        bigkeynum = allkeynums[i]
+        keynum = bigkeynum - startnum
+        print(f"recombining directory {dirs[i]} and {dirs[i+1]}, len {keynum}")
+
+
+
+
+        for j in range(keynum, len(images_current)):
+            alpha = (j - keynum) / (len(images_current) - keynum)
+            image1_path = os.path.join(current_dir, images_current[j])
+            next_image_index = j - keynum if j - keynum < len(images_next) else len(images_next) - 1
+            image2_path = os.path.join(next_dir, images_next[next_image_index])
+
+            image1 = Image.open(image1_path)
+            image2 = Image.open(image2_path)
+
+            blended_image = crossfade_images(image1, image2, alpha)
+            output_images.append(np.array(blended_image))
+            # blended_image.save(os.path.join(output_folder, f"{dirs[i]}_{dirs[i+1]}_crossfade_{j:04}.png"))
+
+    final_dir = os.path.join(root_folder, dirs[-1])
+    for c in range(allkeynums[-1], len(final_dir)):
+
+        images_final = sorted(os.listdir(current_dir))
+        image1_path = os.path.join(current_dir, images_final[c])
+        image1 = Image.open(image1_path)
+        output_images.append(np.array(image1))
+
+
+
+    output_save_location = os.path.join(output_folder, "crossfade.mp4")
+    generated_vid = bmethod.pil_images_to_video(output_images, output_save_location, fps)
+    return generated_vid
+
+def getkeynums (folder_path):
+    filenames = os.listdir(folder_path)
+
+    # Filter filenames to keep only the ones starting with "keys" and ending with ".png"
+    keys_filenames = [f for f in filenames if f.startswith("keys") and f.endswith(".png")]
+
+    # Sort the filtered filenames
+    sorted_keys_filenames = sorted(keys_filenames, key=lambda f: int(re.search(r'(\d+)', f).group(0)))
+
+    # Extract the numbers from the sorted filenames
+    return [int(re.search(r'(\d+)', f).group(0)) for f in sorted_keys_filenames]
+
+
+def get_num_at_index(folder_path,index):
+    """Get the starting number of the output images in a folder."""
+    filenames = os.listdir(folder_path)
+
+    # Filter filenames to keep only the ones starting with "keys" and ending with ".png"
+    #keys_filenames = [f for f in filenames if f.startswith("keys") and f.endswith(".png")]
+
+    # Sort the filtered filenames
+    sorted_keys_filenames = sorted(filenames, key=lambda f: int(re.search(r'(\d+)', f).group(0)))
+
+    # Extract the numbers from the sorted filenames
+    numbers = [int(re.search(r'(\d+)', f).group(0)) for f in sorted_keys_filenames]
+    return numbers[index]