Try this.
It's very simple, and it won't keep track of which textbox you remove.
Private btnID As Integer = 1
Private txtTop As Integer = firsttextBox.Top
Private Sub addButton_Click(ByVal sender As Object, ByVal e As EventArgs) Handles addButton.Click
Dim txtBox As New TextBox
txtBox.Name = "txt" & buttonID
txtBox.Width = firstTextBox.Width
txtBox.Top = txtTop + txtBox.Height + 3
txtBox.Left = firsttextBox.Left
txtTop = txtBox.Top
Me.Controls.Add(txtTop)
Dim btnRemove As New Button
btnRemove.Name = "btn" & buttonID
btnRemove.Width = addButton.Width
btnRemove.Text = "Remove"
btnRemove.Top = txtBox.Top
btnRemove.Left = addButton.Left
AddHandler btnRemove.Click, AddressOf removeBtnTxtBox_Click
Me.Controls.Add(txtTop)
btnID += 1
End Sub
Private Sub removeBtnTxtBox_Click(ByVal sender As Object, ByVal e As EventArgs)
Dim ID As Integer = DirectCast(sender, Button).Name.SubString(3, 1)
Me.Controls.RemoveByKey("txt" & ID)
Me.Controls.RemoveByKey("btn" & ID)
End Sub