Skip to content

GitLab

Commit 035b5ae8 authored by Andy Regensky's avatar Andy Regensky
Browse files

Implement workers for background computations and perform on dedicated threads

parent 53c42f2f
Hide whitespace changes
Inline Side-by-side
Showing with 358 additions and 179 deletions
fishui/MainViewModel.py
View file @ 035b5ae8
import json
from PySide2.QtCore import QObject, Signal, QThreadPool
from PySide2.QtCore import QObject, Signal, QThreadPool, Slot
import numpy as np
import imageio
from fishui import projections
from fishui import coordinate_conversion
from fishui.Worker import Worker
from fishui.workers import BlockProjectionWorker, InterpolationWorker
class MainViewModel(QObject):
......@@ -14,6 +14,12 @@ class MainViewModel(QObject):
def __init__(self, parent=None):
super().__init__(parent)
self.bp_threadpool = QThreadPool(self)
self.bp_threadpool.setMaxThreadCount(1)
self.ip_threadpool = QThreadPool(self)
self.ip_threadpool.setMaxThreadCount(1)
self.projections = {
'equisolid': projections.EquisolidProjection,
'equidistant': projections.EquidistantProjection,
......@@ -42,7 +48,6 @@ class MainViewModel(QObject):
self.projection = None
self.perspective = None
self.fov_180_radius = None
self.center = None
self.block_xf = None
self.block_yf = None
self.block_xp = None
......@@ -59,11 +64,14 @@ class MainViewModel(QObject):
self.updateProjections()
# Background images
self.threadpool = QThreadPool(self)
self._fisheye_images = []
self._perspective_images = []
self._perspective_images = {}
self._current_idx = 0
@property
def center(self):
return (np.asarray(self.sensor_size_px) - 1) / 2
@property
def sensor_dimensions_valid(self):
return np.isclose(self.sensor_size_px[0]/self.sensor_size_mm[0],
......@@ -72,17 +80,17 @@ class MainViewModel(QObject):
def setSensorSize(self, sensor_size):
self.sensor_size_mm = sensor_size
self.updateProjections()
self.updatePerspectiveImagesAsync()
self.updatePerspectiveImages()
def setSensorResolution(self, resolution):
self.sensor_size_px = resolution
self.updateProjections()
self.updatePerspectiveImagesAsync()
self.updatePerspectiveImages()
def setFocalLength(self, focal_length_mm):
self.focal_length_mm = focal_length_mm
self.updateProjections()
self.updatePerspectiveImagesAsync()
self.updatePerspectiveImages()
def setProjection(self, projection_key, calib_file=None):
if projection_key == 'calibrated' and calib_file is not None:
......@@ -93,7 +101,7 @@ class MainViewModel(QObject):
raise Exception("Calibration file argument is valid for calibrated projection only.")
self.projection_key = projection_key
self.updateProjections()
self.updatePerspectiveImagesAsync()
self.updatePerspectiveImages()
def setOrigin(self, origin):
self.origin = origin
......@@ -144,133 +152,40 @@ class MainViewModel(QObject):
self.projection = self.projections[self.projection_key](focal_length_px)
self.perspective = projections.PerspectiveProjection(focal_length_px)
self.fov_180_radius = self.projection.radius(np.deg2rad(90))
self.center = (np.asarray(self.sensor_size_px) - 1) / 2
self.blockSignals(True)
self.updateFisheyeBlockpoints()
self.updatePerspectiveBlockpoints()
self.updatePerspectiveMovedBlockpoints()
self.updateReprojectedBlockpoints()
self.blockSignals(False)
self.didChange.emit()
def updateFisheyeBlockpoints(self):
block_x = np.zeros(4 * self.samples_per_edge)
block_y = np.zeros(4 * self.samples_per_edge)
# Top edge.
block_x[0:self.samples_per_edge] = np.linspace(self.origin[0], self.origin[0] + self.blocksize[0], self.samples_per_edge)
block_y[0:self.samples_per_edge] = self.origin[1]
# Right edge.
block_x[self.samples_per_edge:2 * self.samples_per_edge] = self.origin[0] + self.blocksize[0]
block_y[self.samples_per_edge:2 * self.samples_per_edge] = np.linspace(self.origin[1], self.origin[1] + self.blocksize[1], self.samples_per_edge)
# Bottom edge.
block_x[2 * self.samples_per_edge:3 * self.samples_per_edge] = np.linspace(self.origin[0] + self.blocksize[0], self.origin[0],
self.samples_per_edge)
block_y[2 * self.samples_per_edge:3 * self.samples_per_edge] = self.origin[1] + self.blocksize[1]
# Left edge.
block_x[3 * self.samples_per_edge:4 * self.samples_per_edge] = self.origin[0]
block_y[3 * self.samples_per_edge:4 * self.samples_per_edge] = np.linspace(self.origin[1] + self.blocksize[1], self.origin[1],
self.samples_per_edge)
# Filter within FOV 180
r, _ = coordinate_conversion.cartesian_to_polar(block_y - self.center[1],
block_x - self.center[0])
block_x = block_x[r <= self.fov_180_radius]
block_y = block_y[r <= self.fov_180_radius]
self.block_xf = block_x
self.block_yf = block_y
self.didChange.emit()
def updatePerspectiveBlockpoints(self):
block_yfc = self.block_yf - self.center[1]
block_xfc = self.block_xf - self.center[0]
r_f, phi = coordinate_conversion.cartesian_to_polar(block_yfc, block_xfc)
r_p = self.projection.transform_to(self.perspective, r_f)
self.uwc_sign = np.sign(r_p)
block_ypc, block_xpc = coordinate_conversion.polar_to_cartesian(r_p, phi)
self.block_yp = block_ypc
self.block_xp = block_xpc
self.didChange.emit()
def updatePerspectiveMovedBlockpoints(self):
if self.affineModel:
self.updateControlBlockpoints()
self.updatePerspectiveMovedBlockpointsAffine()
else:
self.block_ypm = self.block_yp + self.motion_vector_cp0[1] * self.uwc_sign
self.block_xpm = self.block_xp + self.motion_vector_cp0[0] * self.uwc_sign
self.didChange.emit()
def updateReprojectedBlockpoints(self):
r_pm, phi_m = coordinate_conversion.cartesian_to_polar(self.block_ypm, self.block_xpm)
phi_m = phi_m - (self.uwc_sign < 0) * np.pi
r_fm = self.perspective.transform_to(self.projection, r_pm)
r_fm = r_fm + (self.uwc_sign < 0) * 2 * (self.projection.radius(np.deg2rad(90)) - r_fm)
block_yfm, block_xfm = coordinate_conversion.polar_to_cartesian(r_fm, phi_m)
self.block_yfm = block_yfm + self.center[1]
self.block_xfm = block_xfm + self.center[0]
self.didChange.emit()
def updateControlBlockpoints(self):
block_center = np.asarray(self.origin) + (np.asarray(self.blocksize) - 1) / 2
r_f, phi = coordinate_conversion.cartesian_to_polar(block_center[1] - self.center[1],
block_center[0] - self.center[0])
r_p = self.projection.transform_to(self.perspective, r_f)
cp0_y, cp0_x = coordinate_conversion.polar_to_cartesian(r_p, phi)
cp0_y = cp0_y - self.blocksize[1] / 2
cp0_x = cp0_x - self.blocksize[0] / 2
self.block_ctr_px = np.array([cp0_x, cp0_x + self.blocksize[0], cp0_x + self.blocksize[0], cp0_x])
self.block_ctr_py = np.array([cp0_y, cp0_y, cp0_y + self.blocksize[1], cp0_y + self.blocksize[1]])
self.didChange.emit()
def updatePerspectiveMovedBlockpointsAffine(self):
cp0_position = (self.block_ctr_py[0], self.block_ctr_px[0])
cps_motion_vector = np.array([self.motion_vector_cp0[0], self.motion_vector_cp1[0],
self.motion_vector_cp0[1], self.motion_vector_cp1[1]])
block_yp_cp0 = self.block_yp - cp0_position[0]
block_xp_cp0 = self.block_xp - cp0_position[1]
block_ypm = np.empty_like(self.block_yp)
block_xpm = np.empty_like(self.block_xp)
for i in range(len(self.block_xp)):
m = block_yp_cp0[i] / (self.blocksize[0] - 1)
n = block_xp_cp0[i] / (self.blocksize[1] - 1)
affine_matrix = np.array([[-m, m, 1 - n, n],
[1 - n, n, m, -m]])
mv = affine_matrix.dot(cps_motion_vector)
block_ypm[i] = self.block_yp[i] + mv[0]
block_xpm[i] = self.block_xp[i] + mv[1]
block_ctr_pxm = np.empty(4)
block_ctr_pym = np.empty(4)
for i in range(4):
m = (self.block_ctr_py[i] - cp0_position[0]) / (self.blocksize[0] - 1)
n = (self.block_ctr_px[i] - cp0_position[1]) / (self.blocksize[1] - 1)
affine_matrix = np.array([[-m, m, 1 - n, n],
[1 - n, n, m, -m]])
mv = affine_matrix.dot(cps_motion_vector)
block_ctr_pym[i] = self.block_ctr_py[i] + mv[0]
block_ctr_pxm[i] = self.block_ctr_px[i] + mv[1]
self.block_ypm = block_ypm
self.block_xpm = block_xpm
self.block_ctr_pym = block_ctr_pym
self.block_ctr_pxm = block_ctr_pxm
self.bp_threadpool.clear()
worker = BlockProjectionWorker(self.perspective,
self.projection,
self.samples_per_edge,
self.origin,
self.blocksize,
self.center,
self.motion_vector_cp0,
self.motion_vector_cp1,
self.affineModel)
worker.signals.didFinish.connect(self.blockProjectionDidFinish)
self.bp_threadpool.start(worker)
@Slot(BlockProjectionWorker.Result)
def blockProjectionDidFinish(self, result: BlockProjectionWorker.Result):
self.block_yf = result.block_yf
self.block_xf = result.block_xf
self.block_yp = result.block_yp
self.block_xp = result.block_xp
self.block_ypm = result.block_ypm
self.block_xpm = result.block_xpm
self.block_yfm = result.block_yfm
self.block_xfm = result.block_xfm
self.block_ctr_py = result.block_ctr_py
self.block_ctr_px = result.block_ctr_px
self.block_ctr_pym = result.block_ctr_pym
self.block_ctr_pxm = result.block_ctr_pxm
self.didChange.emit()
def setFisheyeImages(self, images):
self._fisheye_images = images
self._current_idx = 0
self.blockSignals(True)
self.updatePerspectiveImagesAsync()
self.blockSignals(False)
self.updatePerspectiveImages()
self.didChange.emit()
def nextFisheyeImage(self):
......@@ -291,44 +206,29 @@ class MainViewModel(QObject):
def currentPerspectiveImage(self):
if len(self._perspective_images) == 0:
return None
return self._perspective_images[self._current_idx % len(self._perspective_images)][
self.perspective_scale - 1]
def updatePerspectiveImagesAsync(self):
worker = Worker(self.updatePerspectiveImages)
self.threadpool.start(worker)
key = (self.currentFisheyeImage, self.perspective_scale)
if key in self._perspective_images:
return self._perspective_images[key]
else:
None
def updatePerspectiveImages(self):
self._perspective_images = []
self._perspective_images = {}
self.didChange.emit()
perspective_images = []
for fisheye_image in self._fisheye_images:
image = imageio.imread(fisheye_image)[:, :, :3]
scale_images = []
self.ip_threadpool.clear()
for image_file in self._fisheye_images:
for scale in range(1, 11):
scale_images.append(self.interpolatePerspectiveImage(image, scale))
perspective_images.append(scale_images)
self._perspective_images = perspective_images
worker = InterpolationWorker(image_file, scale, self.perspective, self.projection, self.center)
worker.signals.didFinish.connect(self.interpolationDidFinish)
self.ip_threadpool.start(worker)
@Slot(InterpolationWorker.Result)
def interpolationDidFinish(self, result: InterpolationWorker.Result):
self._perspective_images[(result.image_file, result.scale)] = result.interpolated_image
self.didChange.emit()
def interpolatePerspectiveImage(self, fisheye_image, scale):
yp, xp = np.mgrid[:fisheye_image.shape[0], :fisheye_image.shape[1]]
yp = yp - self.center[1]
xp = xp - self.center[0]
yp = yp * scale
xp = xp * scale
rp, phi = coordinate_conversion.cartesian_to_polar(yp, xp)
rf = self.projection.radius(self.perspective.theta(rp))
yf, xf = coordinate_conversion.polar_to_cartesian(rf, phi)
yf = yf + self.center[1]
xf = xf + self.center[0]
# Nearest neighbor
ynn = np.round(yf).astype(np.int)
xnn = np.round(xf).astype(np.int)
perspective_image = np.zeros(fisheye_image.shape, dtype=fisheye_image.dtype)
mask_low = np.logical_and(0 <= ynn, 0 <= xnn)
mask_high = np.logical_and(ynn < fisheye_image.shape[0], xnn < fisheye_image.shape[1])
mask = np.logical_and(mask_low, mask_high)
perspective_image[mask] = fisheye_image[ynn[mask], xnn[mask]]
return perspective_image
def deleteLater(self):
self.bp_threadpool.clear()
self.ip_threadpool.clear()
fishui/MainWindow.py
View file @ 035b5ae8
......@@ -32,3 +32,6 @@ class MainWindow(QMainWindow):
self.viewModel.nextFisheyeImage()
elif event.key() == Qt.Key_Left:
self.viewModel.previousFisheyeImage()
def closeEvent(self, event):
self.viewModel.deleteLater()
fishui/widgets/FisheyeBlockWidget.py
View file @ 035b5ae8
......@@ -14,7 +14,7 @@ class FisheyeBlockWidget(QWidget):
self.fisheyePolygon = None
self.reprojectPolygon = None
self.updatePolygons(repaint=False)
self.updatePolygons(update=False)
self.initialOrigin = None
self.initialMouse = None
......@@ -22,16 +22,18 @@ class FisheyeBlockWidget(QWidget):
self.show()
@Slot()
def updatePolygons(self, repaint=True):
def updatePolygons(self, update=True):
fisheyePolygon = QPolygonF()
reprojectPolygon = QPolygonF()
if self.viewModel.block_xf is None:
return
for idx in range(len(self.viewModel.block_xf)):
fisheyePolygon.append(QPointF(self.viewModel.block_xf[idx], self.viewModel.block_yf[idx]))
reprojectPolygon.append(QPointF(self.viewModel.block_xfm[idx], self.viewModel.block_yfm[idx]))
self.fisheyePolygon = fisheyePolygon
self.reprojectPolygon = reprojectPolygon
if repaint:
self.repaint()
if update:
self.update()
def sizeHint(self):
return QSize(400, 400)
......@@ -54,12 +56,14 @@ class FisheyeBlockWidget(QWidget):
pen = QPen(blue)
pen.setWidth(painter.window().width()/200)
painter.setPen(pen)
painter.drawPolygon(self.reprojectPolygon)
if self.reprojectPolygon:
painter.drawPolygon(self.reprojectPolygon)
pen = QPen(orange)
pen.setWidth(painter.window().width() / 200)
painter.setPen(pen)
painter.drawPolygon(self.fisheyePolygon)
if self.fisheyePolygon:
painter.drawPolygon(self.fisheyePolygon)
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
......
fishui/widgets/PerspectiveBlockWidget.py
View file @ 035b5ae8
import sys
from PySide2.QtGui import QPainter, QColor, QPen, QPalette, QPolygonF
from PySide2.QtWidgets import QWidget, QApplication
from PySide2.QtWidgets import QWidget
from PySide2.QtCore import QSize, Qt, QPointF, QRect, Slot
from fishui.MainViewModel import MainViewModel
......@@ -18,7 +18,7 @@ class PerspectiveBlockWidget(QWidget):
self.perspectiveMovedPolygon = None
self.controlPointPolygon = None
self.controlPointMovedPolygon = None
self.updatePolygon(repaint=False)
self.updatePolygon(update=False)
self.initialMV0 = None
self.initialMV1 = None
......@@ -28,7 +28,9 @@ class PerspectiveBlockWidget(QWidget):
self.show()
@Slot()
def updatePolygon(self, repaint=True):
def updatePolygon(self, update=True):
if self.viewModel.block_xp is None:
return
perspectivePolygon = QPolygonF()
perspectiveMovedPolygon = QPolygonF()
controlPointPolygon = QPolygonF()
......@@ -54,8 +56,8 @@ class PerspectiveBlockWidget(QWidget):
controlPointMovedPolygon.append(QPointF(ctr_pxm[idx], ctr_pym[idx]))
self.controlPointPolygon = controlPointPolygon
self.controlPointMovedPolygon = controlPointMovedPolygon
if repaint:
self.repaint()
if update:
self.update()
def sizeHint(self):
return QSize(400, 400)
......@@ -78,12 +80,14 @@ class PerspectiveBlockWidget(QWidget):
pen = QPen(blue)
pen.setWidth(painter.window().width() / 200)
painter.setPen(pen)
painter.drawPolygon(self.perspectiveMovedPolygon)
if self.perspectiveMovedPolygon:
painter.drawPolygon(self.perspectiveMovedPolygon)
pen = QPen(orange)
pen.setWidth(painter.window().width() / 200)
painter.setPen(pen)
painter.drawPolygon(self.perspectivePolygon)
if self.perspectivePolygon:
painter.drawPolygon(self.perspectivePolygon)
def drawControlPointBlock(self, event, painter):
color = QColor(0, 0, 0)
......@@ -91,8 +95,10 @@ class PerspectiveBlockWidget(QWidget):
pen.setColor(color)
pen.setWidth(painter.window().width()/400)
painter.setPen(pen)
painter.drawPolygon(self.controlPointPolygon)
painter.drawPolygon(self.controlPointMovedPolygon)
if self.controlPointPolygon:
painter.drawPolygon(self.controlPointPolygon)
if self.controlPointMovedPolygon:
painter.drawPolygon(self.controlPointMovedPolygon)
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
......
fishui/workers/BlockProjectionWorker.py 0 → 100644
View file @ 035b5ae8
from PySide2.QtCore import QObject, QRunnable, Signal, Slot, QMutex, QMutexLocker
from fishui import coordinate_conversion
import numpy as np
class BlockProjectionWorker(QRunnable):
class Result:
def __init__(self,
block_yf, block_xf,
block_yp, block_xp,
block_ypm, block_xpm,
block_yfm, block_xfm,
block_ctr_py, block_ctr_px,
block_ctr_pym, block_ctr_pxm):
self.block_yf = block_yf
self.block_xf = block_xf
self.block_yp = block_yp
self.block_xp = block_xp
self.block_ypm = block_ypm
self.block_xpm = block_xpm
self.block_yfm = block_yfm
self.block_xfm = block_xfm
self.block_ctr_py = block_ctr_py
self.block_ctr_px = block_ctr_px
self.block_ctr_pym = block_ctr_pym
self.block_ctr_pxm = block_ctr_pxm
def __init__(self,
perspective,
projection,
samples_per_edge,
origin,
blocksize,
center,
motion_vector_cp0,
motion_vector_cp1,
affineModel):
super().__init__()
self.perspective = perspective
self.projection = projection
self.samples_per_edge = samples_per_edge
self.origin = origin
self.blocksize = blocksize
self.center = center
self.motion_vector_cp0 = motion_vector_cp0
self.motion_vector_cp1 = motion_vector_cp1
self.affineModel = affineModel
self.signals = BlockProjectionWorkerSignals()
self.block_yf = None
self.block_xf = None
self.block_yp = None
self.block_xp = None
self.block_ypm = None
self.block_xpm = None
self.block_yfm = None
self.block_xfm = None
self.uwc_sign = None
self.block_ctr_py = None
self.block_ctr_px = None
self.block_ctr_pym = None
self.block_ctr_pxm = None
@Slot()
def run(self):
self.updateFisheyeBlockpoints()
self.updatePerspectiveBlockpoints()
self.updatePerspectiveMovedBlockpoints()
self.updateReprojectedBlockpoints()
result = self.Result(self.block_yf, self.block_xf,
self.block_yp, self.block_xp,
self.block_ypm, self.block_xpm,
self.block_yfm, self.block_xfm,
self.block_ctr_py, self.block_ctr_px,
self.block_ctr_pym, self.block_ctr_pxm)
self.signals.didFinish.emit(result)
# perform projections
def updateFisheyeBlockpoints(self):
block_x = np.zeros(4 * self.samples_per_edge)
block_y = np.zeros(4 * self.samples_per_edge)
# Top edge.
block_x[0:self.samples_per_edge] = np.linspace(self.origin[0],
self.origin[0] + self.blocksize[0],
self.samples_per_edge)
block_y[0:self.samples_per_edge] = self.origin[1]
# Right edge.
block_x[self.samples_per_edge:2 * self.samples_per_edge] = self.origin[0] + self.blocksize[0]
block_y[self.samples_per_edge:2 * self.samples_per_edge] = np.linspace(self.origin[1],
self.origin[1] + self.blocksize[1],
self.samples_per_edge)
# Bottom edge.
block_x[2 * self.samples_per_edge:3 * self.samples_per_edge] = np.linspace(self.origin[0] + self.blocksize[0],
self.origin[0],
self.samples_per_edge)
block_y[2 * self.samples_per_edge:3 * self.samples_per_edge] = self.origin[1] + self.blocksize[1]
# Left edge.
block_x[3 * self.samples_per_edge:4 * self.samples_per_edge] = self.origin[0]
block_y[3 * self.samples_per_edge:4 * self.samples_per_edge] = np.linspace(self.origin[1] + self.blocksize[1],
self.origin[1],
self.samples_per_edge)
# Filter within FOV 180
rf_90 = self.projection.radius(np.deg2rad(90))
rf, _ = coordinate_conversion.cartesian_to_polar(block_y - self.center[1],
block_x - self.center[0])
block_x = block_x[rf <= rf_90]
block_y = block_y[rf <= rf_90]
self.block_yf = block_y
self.block_xf = block_x
def updatePerspectiveBlockpoints(self):
block_yfc = self.block_yf - self.center[1]
block_xfc = self.block_xf - self.center[0]
r_f, phi = coordinate_conversion.cartesian_to_polar(block_yfc, block_xfc)
r_p = self.projection.transform_to(self.perspective, r_f)
self.uwc_sign = np.sign(r_p)
block_ypc, block_xpc = coordinate_conversion.polar_to_cartesian(r_p, phi)
self.block_yp = block_ypc
self.block_xp = block_xpc
def updatePerspectiveMovedBlockpoints(self):
if self.affineModel:
self.updateControlBlockpoints()
self.updatePerspectiveMovedBlockpointsAffine()
else:
self.block_ypm = self.block_yp + self.motion_vector_cp0[1] * self.uwc_sign
self.block_xpm = self.block_xp + self.motion_vector_cp0[0] * self.uwc_sign
def updateReprojectedBlockpoints(self):
r_pm, phi_m = coordinate_conversion.cartesian_to_polar(self.block_ypm, self.block_xpm)
phi_m = phi_m - (self.uwc_sign < 0) * np.pi
r_fm = self.perspective.transform_to(self.projection, r_pm)
# FIXME: Use NEW UWC
rf_90 = self.projection.radius(np.deg2rad(90))
r_fm = r_fm + (self.uwc_sign < 0) * 2 * (rf_90 - r_fm)
block_yfm, block_xfm = coordinate_conversion.polar_to_cartesian(r_fm, phi_m)
self.block_yfm = block_yfm + self.center[1]
self.block_xfm = block_xfm + self.center[0]
def updateControlBlockpoints(self):
block_center = np.asarray(self.origin) + (np.asarray(self.blocksize) - 1) / 2
r_f, phi = coordinate_conversion.cartesian_to_polar(block_center[1] - self.center[1],
block_center[0] - self.center[0])
r_p = self.projection.transform_to(self.perspective, r_f)
cp0_y, cp0_x = coordinate_conversion.polar_to_cartesian(r_p, phi)
cp0_y = cp0_y - self.blocksize[1] / 2
cp0_x = cp0_x - self.blocksize[0] / 2
self.block_ctr_px = np.array([cp0_x,
cp0_x + self.blocksize[0],
cp0_x + self.blocksize[0],
cp0_x])