E cogido tu programa y le cambie el codigo de operar, a mi me funciona ay te lo dejo
Private Sub cmdBorrar_Click()
txtRaiz1.Text = ""
txtRaiz2.Text = ""
End Sub
Private Sub cmdOperar_Click()
Dim a As Integer, b As Integer, c As Integer
a = Val(txtNum1.Text)
b = Val(txtNum2.Text)
c = Val(txtNum3.Text)
txtRaiz1.Text = -b + Sqr((b * b) - (4 * a * c)) / 2 * a
txtRaiz1.Text = -b - Sqr((b * b) - (4 * a * c)) / 2 * a
If txtRaiz1.Text = txtRaiz2.Text Then
txtRaiz2.Text = ""
End If
End Sub
Private Sub cmdSalir_Click()
End
End Sub