I found a workaround: Creating an image file as mask and set it to the camera. But perhaps it is possible to set the rectangle directly?
from PIL import Image, ImageDraw
def createFullMaskFor(camera):
width = int(camera.photo.meta["File/ImageWidth"])
height = int(camera.photo.meta["File/ImageHeight"])
img = Image.new("RGB", (width, height))
draw = ImageDraw.Draw(img)
draw.rectangle(((0, 0), (width, height)), fill="black")
name = "MASK.JPEG"
img.save(name, "JPEG")
newMask = Metashape.Mask()
newMask.load(name)
camera.mask = newMask