One option is to pass in an error handler to your DirectoryInfo and FileSystemAccessRule enumeration methods.  Those methods can catch all errors and pass then to the handler, which can optionally handle the error and allow traversal to continue:
Public Class DirectoryExtensions
    Shared Function GetFileSystemAccessRules(d As DirectoryInfo, ByVal errorHandler As Action(Of Object, DirectoryTraversalErrorEventArgs)) As IEnumerable(Of FileSystemAccessRule)
        Try
            Dim ds As DirectorySecurity = d.GetAccessControl()
            Dim arrRules As AuthorizationRuleCollection = ds.GetAccessRules(True, True, GetType(Security.Principal.NTAccount))
            Return arrRules.Cast(Of FileSystemAccessRule)()
        Catch ex As Exception
            If (Not HandleError(errorHandler, d.FullName, ex))
                Throw
            End If
            Return Enumerable.Empty(Of FileSystemAccessRule)()
        End Try
    End Function
    
    Shared Function EnumerateDirectories(ByVal directory As String, ByVal errorHandler As Action(Of Object, DirectoryTraversalErrorEventArgs)) As IEnumerable(Of DirectoryInfo)
        Dim di As DirectoryInfo
        Try
            di = new DirectoryInfo(directory)
        Catch ex As Exception
            If (Not HandleError(errorHandler, directory, ex))
                Throw
            End If
            Return Enumerable.Empty(Of DirectoryInfo)()
        End Try
        ' In .NET Core 2.1+ it should be able to recursively enumerate directories and ignore errors as follows:
        ' Dim query = { di }.Concat(di.EnumerateDirectories("*", New System.IO.EnumerationOptions With { .RecurseSubdirectories = True, .IgnoreInaccessible = True })))
        ' In the meantime, it's necessary to manually catch and ignore errors.
        Dim query = RecursiveEnumerableExtensions.Traverse(di, 
            Function(d)
                Try
                    Return d.GetDirectories()
                Catch ex As Exception
                    If (Not HandleError(errorHandler, d.FullName, ex))
                        Throw
                    End If
                    Return Enumerable.Empty(Of DirectoryInfo)()
                End Try
            End Function
        )
        Return query
    End Function
    Shared Function EnumerateDirectoryFileSystemAccessRules(ByVal directory As String, ByVal errorHandler As Action(Of Object, DirectoryTraversalErrorEventArgs)) As IEnumerable(Of Tuple(Of DirectoryInfo, IEnumerable(Of FileSystemAccessRule)))
        Return EnumerateDirectories(directory, errorHandler).Select(Function(d) Tuple.Create(d, GetFileSystemAccessRules(d, errorHandler)))
    End Function
        
    Shared Public Function SerializeFileAccessRules(ByVal directory As String, ByVal errorHandler As Action(Of Object, DirectoryTraversalErrorEventArgs), Optional ByVal formatting As Formatting = Formatting.Indented)
        Dim query = EnumerateDirectoryFileSystemAccessRules(directory, errorHandler).Select(
            Function(tuple) New With {
                .directory = tuple.Item1.FullName,
                .permissions = tuple.Item2.Select(
                    Function(a) New With { 
                        .IdentityReference = a.IdentityReference.ToString(),
                        .AccessControlType = a.AccessControlType.ToString(),
                        .FileSystemRights = a.FileSystemRights.ToString(),
                        .IsInherited = a.IsInherited.ToString()
                    }
                )
            }
        )
        Return JsonConvert.SerializeObject(query, formatting)   
    End Function
                                
    Private Shared Function HandleError(ByVal errorHandler As Action(Of Object, DirectoryTraversalErrorEventArgs), ByVal fullName as String, ByVal ex as Exception) As Boolean
        If (errorHandler Is Nothing)
            Return False
        End If
        Dim args As New DirectoryTraversalErrorEventArgs(fullName, ex)
        errorHandler(GetType(DirectoryExtensions), args)
        return args.Handled
    End Function                                
End Class
Public Class DirectoryTraversalErrorEventArgs 
    Inherits EventArgs
    Private _directory As String
    Private _exception As Exception
    Public Sub New(ByVal directory as String, ByVal exception as Exception)
        Me._directory = directory
        Me._exception = exception
    End Sub
    
    Public Property Handled As Boolean = false
    Public Readonly Property Directory As String
        Get
            Return _directory
        End Get
    End Property
    Public Readonly Property Exception As Exception
        Get
            Return _exception
        End Get
    End Property
End Class
Public Module RecursiveEnumerableExtensions
    ' Translated to vb.net from this answer https://stackoverflow.com/a/60997251/3744182
    ' To https://stackoverflow.com/questions/60994574/how-to-extract-all-values-for-all-jsonproperty-objects-with-a-specified-name-fro
    ' which was rewritten from the answer by Eric Lippert https://stackoverflow.com/users/88656/eric-lippert
    ' to "Efficient graph traversal with LINQ - eliminating recursion" https://stackoverflow.com/questions/10253161/efficient-graph-traversal-with-linq-eliminating-recursion
    Iterator Function Traverse(Of T)(ByVal root As T, ByVal children As Func(Of T, IEnumerable(Of T)), ByVal Optional includeSelf As Boolean = True) As IEnumerable(Of T)
        If includeSelf Then Yield root
        Dim stack = New Stack(Of IEnumerator(Of T))()
        Try
            stack.Push(children(root).GetEnumerator())
            While stack.Count <> 0
                Dim enumerator = stack.Peek()
                If Not enumerator.MoveNext() Then
                    stack.Pop()
                    enumerator.Dispose()
                Else
                    Yield enumerator.Current
                    stack.Push(children(enumerator.Current).GetEnumerator())
                End If
            End While
        Finally
            For Each enumerator In stack
                enumerator.Dispose()
            Next
        End Try
    End Function
End Module
Then call the method and accumulate the errors in a list of errors like so:
Dim errors = New List(Of Tuple(Of String, String))
Dim handler As Action(Of Object, DirectoryTraversalErrorEventArgs) = 
    Sub(sender, e)
        errors.Add(Tuple.Create(e.Directory, e.Exception.Message))
        e.Handled = true
    End Sub
Dim json As String = DirectoryExtensions.SerializeFileAccessRules(curDirectory, handler) 
' Output the JSON and the errors somehow
Console.WriteLine(json)
For Each e In errors
    Console.WriteLine("Error in directory {0}: {1}", e.Item1, e.Item2)
Next
Notes:
- I am using tuples in a couple of places.  Newer versions of VB.NET have a cleaner syntax for tuples, see Tuples (Visual Basic) for details. 
- The code manually traverses the directory hierarchy by stacking calls to - DirectoryInfo.GetDirectories()and trapping errors from each call.
 - In .NET Core 2.1+ it should be possible to recursively enumerate directories and ignore errors by using - DirectoryInfo.EnumerateDirectories(String, EnumerationOptions)as follows:
 - Dim query = { di }.Concat(di.EnumerateDirectories("*", New System.IO.EnumerationOptions With { .RecurseSubdirectories = True, .IgnoreInaccessible = True })))
 - This overload does not exist in .Net Framework 4.8 though. 
Demo fiddle here