The following is an adaptation of your code with corrections and comments:
struct student {
    char name[30];
    char lastName[30];
    char grades[30][5];
    int numberOfGrades;
};
//Note, error incomplete type occurs 
//when using `struct group *s1 = (struct group *)a;`
//will replace with `struct student` 
int compare(const void *a, const void *b) {
    const struct student *s1 = (struct student *)a;
    const struct student *s2 = (struct student *)b;
    
    return strcmp(s1->lastName, s2->lastName);
}
int main()
{
    struct student group[30];//local scope, will not be recognized in 'compare()'                      
    strcpy(group[0].lastName, "Malaska");//string assignments need to use strcpy or similar, not =
    strcpy(group[1].lastName, "Kowalski");
    strcpy(group[2].lastName, "Adamczyk");
    strcpy(group[3].lastName, "Bakiewicz");
    
    int numberOfStudents = 4;
    //sending correct size of each element 
    qsort(group, numberOfStudents, sizeof(group[0]), compare);
    //                             ^^^^^^^^^^^^^^^^^
}
The following is also an adaptation of your code, but with some additional changes to construct that if in scope of what you are learning, should provide a few more examples of readability, use of scope, use of typedef, and dynamic memory allocation and freeing...
typedef struct {  //use of `typedef struct {` rather than 'struct name {` 
    char name[30];
    char lastName[30];
    char grades[30][5];
    int numberOfGrades;
}student_s; //new type, can be used in place of `struct student` anywhere.
int compare(const void *a, const void *b) {
    const student_s *s1 = (student_s *)a;
    const student_s *s2 = (student_s *)b;
    
    return strcmp(s1->lastName, s2->lastName);
}
int main()
{
    int numberOfStudents = 4;//moved to top so can use group
                             //to properly size instances of group
    //dynamic memory allocation, and clean up at bottom
    student_s *group = calloc(numberOfStudents, sizeof *group);
    if(group)//test here to ensure success
    {
        strcpy(group[0].lastName, "Malaska");
        strcpy(group[1].lastName, "Kowalski");
        strcpy(group[2].lastName, "Adamczyk");
        strcpy(group[3].lastName, "Bakiewicz");
        qsort(group, numberOfStudents, sizeof(group[0]), compare);
        //... more code as necessary
        //when finished with 'group', free it.
        free(group);
    }
}