显卡虚拟化部分
待解决bug：
1 vm puse后鼠标图标花的问题。
2 vm鼠标慢的问题。
3 gtk grab keyboard的问题。
4 pulseaudio启动的问题。
1 GVDI工作内容：
a linux内核修改：原生内核升级，gvt-linux内核升级。
b Qemu修改：原生修改，igvt版本修改。
c Libvirt修改：升级到最新。
d 集成显卡和独立显卡不同修改。
e 虚拟机音频方案修改：backend从alsa修改为pulseaudio，frontend从ac97修改为ich6。
重要服务器中相关文件说明（192.168.2.225）
#### 重要升级包列表：
ls /zfsroot/update-list/
# 共享模式GVDI升级包
drwxr-xr-x. 7 root root 4096 5月 21 17:45 gvt-update
# GVDI声音问题升级包
drwxr-xr-x. 2 root root 137 5月 24 10:09 pulse-update
-rw-r--r--. 1 root root 18592626 5月 24 10:02 pulse-update.tar.gz
# 戴尔工作站调试升级包
drwxr-xr-x. 2 root root 165 5月 24 10:26 wx3100-update
-rw-r--r--. 1 root root 51519458 5月 24 10:22 wx3100-update.tar.gz
# 内核编译环境路径：
ll /zfsroot/rpmbuild
# 透传模式内核spec文件：
/zfsroot/rpmbuild/SPECS/kernel-ml-4.11.spec
# 共享模式内核spec文件：
/zfsroot/rpmbuild/SPECS/kernel-ml-4.14.0.spec
# 透传模式内核源码路径：
/zfsroot/rpmbuild/BUILD/kernel-ml-4.11.4/linux-4.11.4-1.el7.centos.x86_64/
# 共享模式内核源码路径：
/zfsroot/rpmbuild/BUILD/kernel-ml-4.14.10/linux-4.14.0-1.el7.centos.x86_64+/
# 编译后的RPM包路径：
/zfsroot/rpmbuild/RPMS/x86_64/
# 透传模式Qemu源码路径：
/zfsroot/workspace/qemu-dev/qemu
# 共享模式Qemu源码路径：
/zfsroot/workspace/qemu-dev/igvtg-qemu/
# 共享模式Libvirt源码路径：
/zfsroot/workspace/libvirt-dev/libvirt
# 桌面流spice gtk源码路径：
/zfsroot/workspace/spice-gtk-0.22-frank-x86-intel
GVDI终端重要文件路径
# 登录软件配置路径：
ls /etc/lightning_client/
-rw-r--r--. 1 root root 884 4月 27 16:10 client.conf
-rw-r--r--. 1 root root 884 5月 24 09:22 client.conf.bak
-rw-r--r--. 1 root root 105 2月 28 14:41 login.conf
-rw-r--r--. 1 root root 105 5月 24 09:22 login.conf.bak
-rw-r--r--. 1 root root 0 4月 27 16:10 update_kernel
-rw-r--r--. 1 root root 4663 2月 28 11:13 vm_idv_ufi.xml
-rw-r--r--. 1 root root 4239 2月 28 11:13 vm_idv.xml
# 透传Qemu软件路径：
ls /opt/qemu-dev/
# 共享模式Qemu软件路径：
ls /opt/qemu-gvt/
# 共享模式libvirt软件路径：
ls /opt/libvirt-dev/
# 桌面流spice gtk软件路径：
ls /opt/spice-dev/
# 虚拟机bios阶段log文件路径：
ls /var/log/vm_bios.log
# 透传模式虚拟机log文件路径：
ls /var/log/libvirt/qemu/
# 共享模式虚拟机log文件路径：
ls /opt/libvirt-dev/var/log/libvirt/qemu/
# 登录界面程序log文件路径：
ls /var/log/lightning_client.log
桌面流spice gtk
# 编译：
ssh root@192.168.2.225
cd /zfsroot/workspace/spice-gtk-0.22-frank-x86-intel
./config.sh
make;make install
# 运行spice：
gtk/spicy -h 192.168.0.237 -p 5900 -w [password] -f
# 测试gstreamer 是否ok：
cd /zfsroot/workspace/gvt-test/gst-h264-test
make
# 运行测试程序：
export DISPLAY=:0;./decoder
GVDI USB设备管控
# 相关文件：
# USB管控策略配置文件：
ls /etc/lightning_client/login.conf
[private_mode]
[[settings]]
pwd = GNBNGNFNFNHN
save_pwd = True
user_name = niuzc
auto_login = False
[[bind_user]]
user = niuzc
hash = {SHA}n6TVu1GBfc8vljzymz27H8sj9Sg=
# 以下是基于用户的usb管控策略的配置
[[usb_filter]]
class_list = ,
vid_pid_list = ,
policy_type = 0
[multiuser_mode]
# 以下是基于终端的usb管控策略的配置
[[usb_filter]]
class_list = 8, 256, 14, 7
vid_pid_list = ,
policy_type = 0
# 相关python源码：
# 负责usb设备的过滤，虚拟机usb设备的拔插
ls /usr/share/lightning_client/usb_filter_policy.py
# 与usb相关的小工具
ls /usr/share/lightning_client/usb_tools.py
# 监控usb设备的拔插事件
ls /usr/share/lightning_client/udev_monitor.py
dump支持efi的vbios方法：
将显卡拔出，插入一台普通PC主板上，设置Bios，修改显卡的启动模式为efi。启动进入linux系统，运行下面命令dump rom：
echo 1 > /sys/bus/pci/devices/[device-id]/rom
cat /sys/bus/pci/devices/[device-id]/rom > vbios.rom
echo 0 > /sys/bus/pci/devices/[device-id]/rom
linux内核编译：
ssh root@192.168.2.225
cd /zfsroot/rpmbuild/
ls BUILD/kernel-ml-4.11.4/linux-4.11.4-1.el7.centos.x86_64/
rpmbuild -bb SPECS/kernel-ml-4.11.spec
ls RPMS/x86_64/kernel-ml-4.11.4-1.el7.centos.x86_64.rpm
ls BUILD/kernel-ml-4.14.10/linux-4.14.0-1.el7.centos.x86_64+
rpm -bb SPECS/kernel-ml-4.14.0.spec
ls RPMS/x86_64/kernel-ml-4.14.0-1.el7.centos.x86_64.rpm
Qemu原生版本编译：
ssh root@192.168.2.225
cd /zfsroot/workspace/qemu-dev/qemu
mkdir build
cd build/
../conf.sh
make;make install
ls /opt/qemu-dev
igvt Qemu编译：
ssh root@192.168.2.225
cd /zfsroot/workspace/igvtg-qemu/
mkdir build
cd build/
../conf.sh
make;make install
ls /opt/qemu-gvt
编译新版本libvirt：
ssh root@192.168.2.225
cd /zfsroot/workspace/libvirt-dev/libvirt
mkdir build
cd build/
../configure --prefix=/opt/libvirt-dev
make;make install
ls /opt/libvirt-dev
Intel vGPU升级脚本：
ssh root@192.168.2.225
cd /zfsroot/mq-update
./update.sh
代码解释：
内核：
gvt linux修改：
1 添加修改edid接口：
git diff drivers/gpu/drm/i915/gvt/kvmgt.c
+// <<< MQ
+static ssize_t vgpu_edid_show(struct device *dev, struct device_attribute *attr, char *buf);
+static ssize_t vgpu_edid_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count);
+// static DEVICE_ATTR(vgpu_edid, (S_IWUSR | S_IRUGO), vgpu_edid_show, vgpu_edid_store);
+static DEVICE_ATTR_RW(vgpu_edid);
+
+static void print_info(struct intel_vgpu *vgpu, const char *buf, size_t count)
+{
+ int i;
+ for (i = 0; i < count; ++i)
+ {
+ gvt_vgpu_err("%02hhX, ", buf[i]);
+ i ++;
+ }
+}
+
+static ssize_t vgpu_edid_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ if (mdev)
+ {
+ struct intel_vgpu *vgpu = (struct intel_vgpu *)
+ mdev_get_drvdata(mdev);
+ gvt_vgpu_err("%s \n", __func__);
+ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+ int port_num = PORT_B;
+ if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+ port_num = PORT_D;
+ struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+ print_info(vgpu, port->edid->edid_block, EDID_SIZE);
+ }
+
+ return sprintf(buf, "\n");
+}
+
+static ssize_t vgpu_edid_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct mdev_device *mdev = mdev_from_dev(dev);
+ if (mdev)
+ {
+ struct intel_vgpu *vgpu = (struct intel_vgpu *)
+ mdev_get_drvdata(mdev);
+ gvt_vgpu_err("%s [%d] \n", __func__, count);
+ if (count == EDID_SIZE)
+ {
+ /* code */
+ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+ int port_num = PORT_B;
+ if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+ port_num = PORT_D;
+ struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+ memcpy(port->edid->edid_block, buf, EDID_SIZE);
+ intel_gvt_ops->vgpu_reset(vgpu);
+ }
+ print_info(vgpu, buf, count);
+ }
+
+ return count;
+}
+// >>> MQ
+
+
+
static struct attribute *intel_vgpu_attrs[] = {
&dev_attr_vgpu_id.attr,
&dev_attr_hw_id.attr,
+ // <<< MQ
+ &dev_attr_vgpu_edid.attr,
+ // >>> MQ
NULL
};
Qemu:
添加简单镜像加密：
git diff block/qcow2.c
Igvtg-qemu:
解决重启后虚拟机无法显示的问题：
git diff hw/vfio/display.c
+static void vfio_display_clean_dmabufs(VFIOPCIDevice *vdev)
+{
+ VFIODMABuf *dmabuf, *tmp;
+
+ QTAILQ_FOREACH_SAFE(dmabuf, &vdev->dmabufs, next, tmp) {
+ QTAILQ_REMOVE(&vdev->dmabufs, dmabuf, next);
+ dpy_gl_release_dmabuf(vdev->display_con, &dmabuf->buf);
+ close(dmabuf->buf.fd);
+ g_free(dmabuf);
+ }
+}
+
+
+int vfio_display_reset(VFIOPCIDevice *vdev)
+{
+ vfio_display_clean_dmabufs(vdev);
+ return 0;
+}
git diff hw/vfio/pci.h
+int vfio_display_reset(VFIOPCIDevice *vdev);
git diff hw/vfio/pci.c
@@ -1989,6 +1989,8 @@ static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
PCIDevice *pdev = &vdev->pdev;
uint16_t cmd;
+ vfio_display_reset(vdev);
+
解决Qemu突然崩溃：
git diff hw/vfio/display.c
@@ -80,7 +92,9 @@ static VFIODMABuf *vfio_display_get_dmabuf(VFIOPCIDevice *vdev,
dmabuf->buf.width, dmabuf->buf.height,
(plane_type == DRM_PLANE_TYPE_PRIMARY)
? "primary" : "cursor");
- abort();
+ // abort();
+ return NULL;
添加检查vGPU图像是否正常：
git diff hmp-commands.hx
-@item mouse_set @var{index}
+@item mouse_set @var{val}
@findex mouse_set
+Show the vIGD info.
+ETEXI
+
+ {
+ .name = "gk_vIGD_info",
+ .args_type = "",
+ .params = "",
+ .help = "get vIGD state",
+ .cmd = hmp_gk_vIGD_info,
+ },
+
+STEXI
+@item gk_vIGD_info @var{val}
+@findex gk_vIGD_info
git diff include/ui/console.h
+void hmp_gk_vIGD_info(Monitor *mon, const QDict *qdict);
git diff ui/gtk.c
+// MQ <<<
+#include "monitor/monitor.h"
+// >>>
+
@@ -170,6 +174,9 @@ struct GtkDisplayState {
VirtualConsole *ptr_owner;
gboolean full_screen;
+#ifdef MQ_ADD
+ gboolean hide_screen;
+#endif
@@ -1326,13 +1333,36 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
}
}
+#ifdef MQ_ADD
+ if (mq_vgpu_state == mq_starting)
+ {
+ s->hide_screen = TRUE;
+ gtk_widget_hide(s->window);
+ }
+#endif
+
static void gd_accel_full_screen(void *opaque)
{
GtkDisplayState *s = opaque;
+#ifdef MQ_ADD
+ if (!s->hide_screen)
+ {
+ /* code */
+ gtk_widget_hide(s->window);
+ s->hide_screen = TRUE;
+ } else {
+ if (mq_vgpu_state == mq_running)
+ {
+ gtk_widget_show(s->window);
+ s->hide_screen = FALSE;
+ }
+ }
+#else
gtk_menu_item_activate(GTK_MENU_ITEM(s->full_screen_item));
+#endif
@@ -2237,12 +2267,18 @@ static void gd_set_keycode_type(GtkDisplayState *s)
}
static gboolean gtkinit;
+#ifdef MQ_ADD
+static GtkDisplayState *mq_s;
+#endif
void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
{
VirtualConsole *vc;
GtkDisplayState *s = g_malloc0(sizeof(*s));
+#ifdef MQ_ADD
+ mq_s = s;
+#endif
@@ -2306,6 +2342,17 @@ void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
gtk_container_add(GTK_CONTAINER(s->window), s->vbox);
gtk_widget_show_all(s->window);
+#ifdef MQ_ADD
+ if (display_opengl)
+ {
+ /* code */
+ gtk_widget_hide(s->window);
+ s->hide_screen = TRUE;
+ } else {
+ s->hide_screen = FALSE;
+ mq_vgpu_state = mq_running;
+ }
+#endif
+void hmp_gk_vIGD_info(Monitor *mon, const QDict *qdict)
+{
+ monitor_printf(mon, "%d\n", mq_vgpu_state);
+}
git diff include/ui/gtk.h
+#ifdef MQ_ADD
+typedef enum VgpuState
+{
+ mq_starting,
+ mq_running,
+} VgpuState;
+extern VgpuState mq_vgpu_state;
+#endif
+
git diff ui/gtk-egl.c
+#ifdef MQ_ADD
+VgpuState mq_vgpu_state = mq_starting;
+#endif
+
@@ -257,6 +261,21 @@ void gd_egl_scanout_flush(DisplayChangeListener *dcl,
return;
}
+#ifdef MQ_ADD
+ if (mq_vgpu_state == mq_starting)
+ {
+ uint32_t tmp = 0;
+ egl_fb *fb = &vc->gfx.guest_fb;
+ egl_mq_fb_read(&tmp, fb, fb->width / 2, fb->height / 2, 1, 1);
+ if (tmp != 0xff000000)
+ {
+ mq_vgpu_state = mq_running;
+ }
+ // fprintf(stderr, "(%x) pixel data\r", tmp);
+ // fflush(stderr);
+ }
+#endif
+
git diff include/ui/egl-helpers.h
+#define MQ_ADD 1
+#ifdef MQ_ADD
+void egl_mq_fb_read(void *dst, egl_fb *src, int x, int y, int w, int h);
+#endif
git diff ui/egl-helpers.c
+#ifdef MQ_ADD
+void egl_mq_fb_read(void *dst, egl_fb *src, int x, int y, int w, int h)
+{
+ glBindFramebuffer(GL_READ_FRAMEBUFFER, src->framebuffer);
+ glReadBuffer(GL_COLOR_ATTACHMENT0_EXT);
+ glReadPixels(x, y, w, h,
+ GL_BGRA, GL_UNSIGNED_BYTE, dst);
+}
+#endif
+
添加显示隐藏虚拟机画面：
git diff hmp-commands.hx
+Show the gtk window.
+ETEXI
+
+ {
+ .name = "gk_gtk_show",
+ .args_type = "",
+ .params = "",
+ .help = "triggle gtk window show",
+ .cmd = hmp_gk_gtk_show,
+ },
+
+STEXI
+@item gk_gtk_show @var{index}
+@findex gk_gtk_show
git diff ui/gtk.c
+
+void hmp_gk_gtk_show(Monitor *mon, const QDict *qdict)
+{
+#ifdef MQ_ADD
+ GtkDisplayState *s = mq_s;
+ VirtualConsole *vc = gd_vc_find_current(s);
+ if (!s->hide_screen)
+ {
+ /* code */
+ gd_ungrab_keyboard(s);
+ gd_ungrab_pointer(s);
+ gd_update_cursor(vc);
+ gtk_widget_hide(s->window);
+ s->hide_screen = TRUE;
+ } else {
+ if (mq_vgpu_state == mq_running)
+ {
+
+ gd_grab_keyboard(vc, "user-request-main-window");
+ gd_grab_pointer(vc, "user-request-main-window");
+
+ gd_update_cursor(vc);
+
+ // if (!s->full_screen) {
+ // gtk_widget_show_all(s->window);
+ // gtk_notebook_set_show_tabs(GTK_NOTEBOOK(s->notebook), FALSE);
+ // gtk_widget_hide(s->menu_bar);
+ // if (vc->type == GD_VC_GFX)
+ // {
+ // gtk_widget_set_size_request(vc->gfx.drawing_area, -1, -1);
+ // }
+ // gtk_window_fullscreen(GTK_WINDOW(s->window));
+ // s->full_screen = TRUE;
+ // }
+
+ gtk_widget_show(s->window);
+ s->hide_screen = FALSE;
+ }
+ }
+#endif
+}
解决虚拟机迁移不成功问题：
git diff include/hw/vfio/vfio-common.h
git diff hw/vfio/pci.c
git diff hw/vfio/common.c
git diff hw/vfio/common.c
GKapp部分
Windows Server 2012 RemoteApp搭建：
搭建域控制器
部署remoteapp
windows server虚拟机服务器ip：192.168.2.248
ADDC ip：192.168.2.230
GKapp Agent架构
gkapp-manager
gkapp-center
gkfb
gokuprotocol
gkappsession：
gokuapp：
# 源码路径：
# gkappsession:
client/gkapp-enc/gkapp-session/
#gokuapp:
client/X11/cli/
# 编译方式：
ssh root@192.168.108.244
cd /zfsroot/gkapp-workspace/freerdp-gk/
# 清除cmake缓存
./c.sh
# 重新生成cmake缓存
./m.sh
# 编译
make
# 打包成升级包
./make_package.sh
# 在当前目录下会生成如下文件夹：
ls gokuapp-2018-05-22/
# 将此文件夹拷贝到目标服务器中，运行install_gokuapp.sh脚本，即可升级相关软件。
GKapp Client架构
https://git.coding.net/mqddb/gokuapp.git
https://git.coding.net/mqddb/rapp.git
https://git.coding.net/mqddb/gkapp-action.git
gkapp排错基本流程：
1 确定windows server remoteapp是否有问题，方法：打开remote app web access网址，如：https://192.168.2.236/RDWeb/Pages/zh-CN/login.aspx，输入用户名，密码看看是否可以正常获取应用列表？如果可以，在win2012中打开获取的rdp文件，如果能够正常启动remote app，则说明windows server remoteapp没有问题。
2 确定windows server remoteapp没有问题后，再确认gkapp-manager和gkapp-center有没有问题，方法：进入gkapp agent，查看log文件：/var/log/gkapp_manager/django.log， /var/log/gkapp_center/center.log。运行命令：systemctl restart gkapp-manager，systemctl restart gkapp-center，重启两个服务。
3 如果到这里，运行还是有问题。则说明gkappsession和gokuapp程序运行有问题，需要查看文件夹：/var/log/gokuapp/中相关的log文件，确定问题出在哪？
