Match pairs of ( and ) and [ and ] using a stack. If there is a mismatch in such a pair of characters OR if the characters are not available(meaning hanging closed parentheses), collect all such guys in array to be skipped later when creating a clean string.
<?php
function filterUnmatchedParentheses($string){
  $opened = [];
  $unOpened = [];
  $skipIdxs = [];
  $length = strlen($string);
  
  for($i = 0; $i < $length; ++$i){
    if($string[ $i ] === '(' || $string[ $i ] === '['){
      $opened[] = $i;
    }else if($string[ $i ] === ')' || $string[ $i ] === ']'){
      $matchFound = false;
      while(count($opened) > 0){
        $idx = array_pop($opened);
        $char = $string[ $idx ];
        if($char == '(' && $string[ $i ] === ')' || $char == '[' && $string[ $i ] === ']'){
          $matchFound = true;
          break;
        }
        $skipIdxs[] = $idx;
      }
      
      if(!$matchFound){
        $unOpened[] = $i;
      }
    }
  }
  
  $skipIdxs = array_flip(array_merge($skipIdxs, $opened, $unOpened));
  
  $res = "";
  
  for($i = 0; $i < $length; ++$i){
    if(isset($skipIdxs[ $i ])) continue;
    $res .= $string[ $i ];
  }
  
  return $res;
}
Online Demo